├── .gitignore ├── .gitmodules ├── LICENSE ├── Makefile ├── README.md ├── build └── cues │ ├── build.clj │ ├── dev.clj │ └── repl.clj ├── deps.edn ├── doc └── Messaging-Semantics.md ├── src └── cues │ ├── controllers.clj │ ├── deps.clj │ ├── error.clj │ ├── log.clj │ ├── queue.clj │ ├── test.clj │ └── util.clj └── test └── cues ├── deps_test.clj └── queue_test.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /.cpcache/ 2 | /data/ 3 | /pom.xml 4 | /target/ -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "make-clj"] 2 | path = make-clj 3 | url = https://github.com/zalky/make-clj.git 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2022 Zalan Kemenczy 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | 3 | version-number = 0.2.1 4 | group-id = io.zalky 5 | artifact-id = cues 6 | description = Queues on cue: persistent blocking queues, processors, and topologies via ChronicleQueue 7 | license = :apache 8 | url = https://github.com/zalky/cues 9 | 10 | include make-clj/Makefile 11 | 12 | test: 13 | clojure -M:test:cues/j17 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Cues 3 | 4 | 5 | 6 | [![Clojars Project](https://img.shields.io/clojars/v/io.zalky/cues?labelColor=blue&color=green&style=flat-square&logo=clojure&logoColor=fff)](https://clojars.org/io.zalky/cues) 7 | 8 | Queues on cue: low-latency persistent blocking queues, processors, and 9 | graphs via Chronicle Queue. 10 | 11 | For when distributed systems like Kafka are too much, durable-queue is 12 | not enough, and both are too slow. 13 | 14 | [Chronicle Queue](https://github.com/OpenHFT/Chronicle-Queue) is a 15 | broker-less queue framework that provides microsecond latencies 16 | (sometimes less) when persisting data to 17 | disk. [Tape](https://github.com/mpenet/tape) is an excellent wrapper 18 | around Chronicle Queue that exposes an idiomatic Clojure API, but does 19 | not provide blocking or persistent tailers. 20 | 21 | **Cues** extends both to provide: 22 | 23 | 1. Persistent _blocking_ queues, persistent tailers, and appenders 24 | 2. Processors for consuming and producing messages 25 | 3. Simple, declarative graphs for connecting processors together via 26 | queues 27 | 4. Brokerless, fault-tolerant _exactly-once_ message delivery 28 | 5. Message metadata 29 | 6. Microsecond latencies (sometimes even less) 30 | 7. Zero-configuration defaults 31 | 8. Not distributed 32 | 33 | By themselves, the blocking queues are similar to what 34 | [durable-queue](https://github.com/clj-commons/durable-queue) 35 | provides, just one or more orders of magnitude faster. They also come 36 | with an API that aligns more closely with the persistence model: you 37 | get queues, tailers, and appenders, and addressable, immutable indices 38 | that you can traverse forwards and backwards. 39 | 40 | The processors and graphs are meant to provide a dead-simple version 41 | of the abstractions you get in a distributed messaging system like 42 | [Kafka](https://kafka.apache.org/). But there are no clusters to 43 | configure, no partitions to worry about, and it is several orders of 44 | magnitude faster. 45 | 46 | Ultimately the goals of Cues are fairly narrow: a minimal DSL for 47 | connecting message processors into graphs using persistent queues in a 48 | non-distributed environment. 49 | 50 | ## Use Cases 51 | 52 | Cues could be used for: 53 | 54 | - Robust, persistent, low-latency communication between threads or 55 | processes 56 | - Anywhere you might use `clojure.core.async` but require a persistent 57 | model 58 | - Prototyping or mocking up a distributed architecture 59 | 60 | To give you a sense of how it scales, from the Chronicle Queue 61 | [FAQ](https://github.com/OpenHFT/Chronicle-Queue/blob/develop/docs/FAQ.adoc): 62 | 63 | > Our largest Chronicle Queue client pulls in up to 100 TB into a 64 | > single JVM using an earlier version of Chronicle Queue. 65 | 66 | As for message limits: 67 | 68 | > The limit is about 1 GB, as of Chronicle 4.x. The practical limit 69 | > without tuning the configuration is about 16 MB. At this point you 70 | > get significant inefficiencies, unless you increase the data 71 | > allocation chunk size. 72 | 73 | ## Contents 74 | 75 | The Cues API could be grouped into two categories: primitives and 76 | graphs. 77 | 78 | The low-level primitives are easy to get started with, but can be 79 | tedious to work with once you start connecting systems together. 80 | 81 | You could just as easily start with the higher-level graph 82 | abstractions that hide most of the boiler-plate, but then you might 83 | want to circle back at some point to understand the underlying 84 | mechanics. 85 | 86 | 1. [Installation](#installation) 87 | 2. [Quick Start](#quick-start) 88 | 3. [Primitives: Queues, Tailers, and Appenders](#queues) 89 | 4. [Processors and Graphs](#processors-graphs) 90 | - [Processors](#processors) 91 | - [Graphs](#graphs) 92 | - [Topic and Type Filters](#filters) 93 | - [Topologies](#topologies) 94 | 5. [Errors and Exactly Once Message Delivery](#errors) 95 | 6. [Queue Configuration](#configuration) 96 | 7. [Queue Metadata](#metadata) 97 | 8. [Utility Functions](#utilities) 98 | 9. [Runway: Developing Stateful Systems](#runway) 99 | 10. [Data Serialization](#serialization) 100 | 11. [Java 11 & 17](#java) 101 | 12. [Chronicle Queue Analytics (disabled by default)](#analytics) 102 | 103 | ## Installation 104 | 105 | Just add the following dependency in your `deps.edn`: 106 | 107 | ```clj 108 | io.zalky/cues {:mvn/version "0.2.1"} 109 | ``` 110 | 111 | If you do not already have SLF4J bindings loaded in your project, 112 | SLF4J will print a warning and fall back on the no-operation (NOP) 113 | bindings. To suppress the warning, simply include the nop bindings 114 | explicitly in your deps: 115 | 116 | ```clj 117 | org.slf4j/slf4j-nop {:mvn/version "2.0.6"} 118 | ``` 119 | 120 | Java compatibility: Chronicle Queue targets [LTS Java releases 8, 11, 121 | and 122 | 17](https://github.com/OpenHFT/OpenHFT/blob/ea/docs/Java-Version-Support.adoc). See 123 | the additional notes on running Chronicle Queue on [Java 11 & 124 | 17](#java). 125 | 126 | ## Quick Start 127 | 128 | It is really easy to get started with queue primitives: 129 | 130 | ```clj 131 | (require '[cues.queue :as q]) 132 | 133 | (def q (q/queue ::queue-id)) 134 | (def a (q/appender q)) 135 | (def t (q/tailer q)) 136 | 137 | (q/write a {:x 1}) 138 | ;; => 139 | 83313775607808 140 | 141 | (q/read!! t) 142 | ;; => 143 | {:x 1} 144 | ``` 145 | 146 | But connecting queues into a system is also straightforward: 147 | 148 | ```clj 149 | (defmethod q/processor ::inc-x 150 | [process {msg :input}] 151 | {:output (update msg :x inc)}) 152 | 153 | (defmethod q/processor ::store-x 154 | [{{db :db} :opts} {msg :input}] 155 | (swap! db assoc (:x msg) msg) 156 | nil) 157 | 158 | (defonce example-db 159 | (atom nil)) 160 | 161 | (defn example-graph 162 | "Connect processors together using ::source and ::tx queues." 163 | [db] 164 | {:id ::example 165 | :processors [{:id ::source} 166 | {:id ::inc-x 167 | :in {:input ::source} 168 | :out {:output ::tx}} 169 | {:id ::store-x 170 | :in {:input ::tx} 171 | :opts {:db db}}]}) 172 | 173 | (def g 174 | (-> (example-graph example-db) 175 | (q/graph) 176 | (q/start-graph!))) 177 | 178 | (q/send! g ::source {:x 1}) 179 | (q/send! g ::source {:x 2}) 180 | 181 | ;; Messages propagate asynchronously through the graph... then: 182 | 183 | @example-db 184 | ;; => 185 | {2 {:x 2} 186 | 3 {:x 3}} 187 | 188 | ;; Inspect the queues that particpate in the graph: 189 | (q/all-graph-messages g) 190 | ;; => 191 | {::source ({:x 1} {:x 2}) 192 | ::tx ({:x 2} {:x 3})} 193 | ``` 194 | 195 | A similar example graph is defined in the 196 | [`cues.build`](https://github.com/zalky/cues/blob/main/build/cues/build.clj) 197 | namespace. To try it, clone this repo and run the following from the 198 | project root: 199 | 200 | ```clj 201 | clojure -X:server:repl 202 | ``` 203 | 204 | Or if you're on Java 11 or 17: 205 | 206 | ```clj 207 | clojure -X:server:repl:cues/j17 208 | ``` 209 | 210 | Then connect your REPL and try the following: 211 | 212 | ```clj 213 | user> (require '[cues.repl :as repl]) 214 | nil 215 | user> (q/send! (repl/graph) {:x 1}) 216 | 83464099463168 217 | user> (q/all-graph-messages (repl/graph)) 218 | {:cues.build/error (), 219 | :cues.build/source ({:x 1}), 220 | :cues.build/tx ({:x 2, :q/meta {:tx/t 83464099463168}})} 221 | ``` 222 | 223 | The rest of this document just covers these two APIs in more detail. 224 | 225 | ## Primitives: Queues, Tailers, and Appenders 226 | 227 | Cues takes the excellent primitives offered by the 228 | [Tape](https://github.com/mpenet/tape) library and extends them to 229 | provide blocking and a couple of other features. 230 | 231 | Queues couldn't be easier to create: 232 | 233 | ```clj 234 | (require '[cues.queue :as q]) 235 | 236 | (def q (q/queue ::queue-id)) 237 | ``` 238 | 239 | The `::queue-id` id uniquely identifies this queue throughout the 240 | system and across restarts. Anywhere you call `(q/queue ::queue-id)` 241 | it will return an object that references the same queue data. 242 | 243 | You can then append messages to the queue with an appender (audible 244 | gasp!): 245 | 246 | ```clj 247 | (def a (q/appender q)) 248 | 249 | (q/write a {:x 1}) 250 | ;; => 251 | 83313775607808 252 | 253 | (q/write a {:x 2}) 254 | ;; => 255 | 83313775607809 256 | ``` 257 | 258 | All messages are Clojure maps. Once the message has been written, 259 | `cues.queue/write` returns its index on the queue. Both the index and 260 | the message are immutable. There is no way to update a message once it 261 | has been written to disk. 262 | 263 | To read the message back you use a tailer: 264 | 265 | ```clj 266 | (def t (q/tailer q)) 267 | 268 | (q/read t) 269 | ;; => 270 | {:x 1} 271 | ``` 272 | 273 | Tailers are stateful: each tailer tracks its position on the queue 274 | that it is tailing. You can get the current _unread_ index of the 275 | tailer: 276 | 277 | ```clj 278 | (q/index t) 279 | ;; => 280 | 83313775607809 281 | ``` 282 | 283 | When it consumes a message from the queue, the tailer advances to the 284 | next index: 285 | 286 | ```clj 287 | (q/read t) 288 | ;; => 289 | {:x 2} 290 | 291 | (q/index t) 292 | ;; => 293 | 83313775607810 294 | ``` 295 | 296 | This tailer is now one index ahead of the last message that we wrote 297 | at `83313775607809`. Note that while indices are guaranteed to 298 | increase monotonically, there is _no guarantee that they are 299 | contiguous_. In general you should avoid code that tries to predict 300 | future indices on the queue. 301 | 302 | Since we have already read two messages there will be no message at 303 | the tailer's current index. If we try another read with this tailer, 304 | it will return `nil`, without advancing: 305 | 306 | ```clj 307 | (q/read t) 308 | ;; => 309 | nil 310 | ``` 311 | 312 | This is because `cues.queue/read` is non-blocking. 313 | 314 | We can use `cues.queue/read!!` to do a blocking read, which will block 315 | the current thread until a message is available at the tailer's 316 | index. Let's do this in another thread so that we can continue using 317 | the REPL: 318 | 319 | ```clj 320 | (def f (future 321 | (let [t (q/tailer q)] 322 | (while true 323 | (let [msg (q/read!! t)] 324 | ;; blocks until there is something to read 325 | (println "message:" (:x msg))))))) 326 | ;; prints 327 | message: 1 328 | message: 2 329 | ``` 330 | 331 | Notice that we created a new tailer in the future thread. This is 332 | because unlike queues, tailers _cannot be shared across threads_. An 333 | error will be thrown if you attempt to read with the same tailer in 334 | more than one thread. 335 | 336 | Also notice the loop immediately printed the first two messages on the 337 | queue. This is because by default all new tailers will start at the 338 | beginning of the queue (we'll see how to change this next). After 339 | reading the first two messages, the tailer blocked on the third 340 | iteration of the loop. 341 | 342 | We can continue adding messages to the queue using the appender: 343 | 344 | ```clj 345 | (q/write a {:x 3}) 346 | ;; => 347 | 83313775607810 348 | ;; prints 349 | message: 3 350 | 351 | (q/write a {:x 4}) 352 | ;; => 353 | 83313775607811 354 | ;; prints 355 | message: 4 356 | ``` 357 | 358 | And Voila! We are using ultra-low latency, persistent messaging to 359 | communicate between threads. How fast is it? 360 | 361 | ```clj 362 | (require '[criterium.core :as b]) 363 | 364 | (b/quick-bench 365 | (q/write a {:x 1})) 366 | 367 | "Evaluation count : 865068 in 6 samples of 144178 calls. 368 | Execution time mean : 690.932746 ns 369 | Execution time std-deviation : 7.290161 ns 370 | Execution time lower quantile : 683.505105 ns ( 2.5%) 371 | Execution time upper quantile : 698.417843 ns (97.5%) 372 | Overhead used : 2.041010 ns" 373 | 374 | (b/quick-bench 375 | (do (q/write a {:x 1}) 376 | (q/read!! t))) 377 | 378 | "Evaluation count : 252834 in 6 samples of 42139 calls. 379 | Execution time mean : 2.389696 µs 380 | Execution time std-deviation : 64.305722 ns 381 | Execution time lower quantile : 2.340823 µs ( 2.5%) 382 | Execution time upper quantile : 2.466880 µs (97.5%) 383 | Overhead used : 2.035620 ns" 384 | ``` 385 | 386 | Quite fast. Of course it will always depend to a large extent on the 387 | size of the messages you are serializing. 388 | 389 | Cues tailers have two other tricks up their sleeves: 390 | 391 | 1. **Persistence**: like queues, tailers can also be persistent. If 392 | you pass a tailer an id at creation, then that tailer's current 393 | index will persist with that queue across runtimes: 394 | 395 | ```clj 396 | (q/tailer q ::tailer-id) 397 | ``` 398 | 399 | Without an id, this tailer would restart from the beginning of the 400 | queue `q` every time you launch the application. 401 | 402 | 2. **Unblocking**: you can additionally pass a tailer an `unblock` 403 | atom. If the value of the atom is ever set to `true`, then the 404 | tailer will no longer block on reads: 405 | 406 | ```clj 407 | (let [unblock (atom nil)] 408 | (q/tailer q ::tailer-id unblock) 409 | ...) 410 | ``` 411 | 412 | This would typically be used to unblock and dispose of blocked 413 | threads. 414 | 415 | Finally there is another blocking read function, `cues.queue/alts!!` 416 | that given a list of tailers, will complete at most _one_ read from 417 | the first tailer that has a message available. `alts!!` returns a two 418 | element tuple: the first element is the tailer that was read from and 419 | the second element is the message that was read: 420 | 421 | ```clj 422 | (q/alts!! [t1 t2 t3]) 423 | ;; => t2 was the first with a message 424 | [t2 {:x 1}] 425 | ``` 426 | 427 | ### Additional Queue Primitive Functions 428 | 429 | Before moving on, let's cover a few more of the low level functions 430 | for working directly with queues, tailers and appenders. 431 | 432 | Instead of moving `:forward` along a queue, you can change the 433 | direction of a tailer: 434 | 435 | ```clj 436 | (q/set-direction t :backward) 437 | ``` 438 | 439 | You can move a tailer to either the start or end of a queue: 440 | 441 | ```clj 442 | (q/to-start t) 443 | 444 | (q/to-end t) 445 | ``` 446 | 447 | You can move a tailer to a _specific_ index: 448 | 449 | ```clj 450 | (q/to-index t 83313775607810) 451 | ;; => 452 | true 453 | 454 | (q/read t) 455 | ;; => 456 | {:x 3} 457 | ``` 458 | 459 | And you can get the last index written to a queue, either from the 460 | queue or an associated appender: 461 | 462 | ```clj 463 | (q/last-index q) 464 | ;; => 465 | 83313775607811 466 | 467 | (q/last-index a) ; appender associated with q 468 | ;; => 469 | 83313775607811 470 | ``` 471 | 472 | Recall that `cue.queue/index` gets the current _unread_ index of a 473 | tailer. Instead you can get the last _read_ index for a tailer: 474 | 475 | ```clj 476 | (q/index t) 477 | ;; => 478 | 83313775607811 479 | 480 | (q/last-read-index t) 481 | ;; => 482 | 83313775607810 483 | ``` 484 | 485 | You can also read a message from a tailer _without_ advancing the 486 | tailer's position: 487 | 488 | ```clj 489 | (q/index t) 490 | ;; => 491 | 83313775607811 492 | 493 | (q/peek t) 494 | ;; => 495 | {:x 4} 496 | 497 | (q/index t) 498 | ;; => 499 | 83313775607811 500 | ``` 501 | 502 | Finally, while tailers are very cheap, they do represent open 503 | resources on the queue. You should close them once they're no longer 504 | needed or it can add up over time. 505 | 506 | To this end, you can either call `q/close-tailer!` directly on the 507 | tailer, or use a scoped tailer constructor: 508 | 509 | ```clj 510 | (q/with-tailer [tailer queue] 511 | ...) 512 | ``` 513 | This gives you a new `tailer` on the given queue that will be closed 514 | when the `with-tailer` block exits scope. 515 | 516 | ## Processors and Graphs 517 | 518 | While the queue primitives are easy to use, they can be tedious to 519 | work with when combining primitives into systems. To this end, Cues 520 | provides two higher-level features: 521 | 522 | 1. Processor functions 523 | 2. A graph DSL to connect processors together via queues 524 | 525 | ### Processors 526 | 527 | Processors are defined using the `cues.queue/processor` multimethod: 528 | 529 | ```clj 530 | (defmethod q/processor ::inc-x 531 | [process {msg :input}] 532 | {:output (update msg :x inc)}) 533 | 534 | (defmethod q/processor ::store-x 535 | [{{db :db} :opts} {msg :input}] 536 | (swap! db assoc (:x msg) msg) 537 | nil) 538 | ``` 539 | 540 | Each processor method has two arguments. The first argument, 541 | `process`, is a roll-up of the processor's execution context and all 542 | the resources that the processor may need to handle messages. We'll 543 | return to this in the following sections. 544 | 545 | The other argument is a map of input messages. Each message is stored 546 | at an input binding, and there can be more than one message in the 547 | map. In the `::inc-x` processor, a single input message is bound to 548 | `:input`. 549 | 550 | Typically a processor will then take the input messages, and return 551 | one or more output messages in an output binding map. Here, 552 | `::inc-x` binds the output message to `:output`. In contrast, 553 | you'll notice that `::store-x` returns `nil`, not a binding map. The 554 | next section will explain why `::inc-x` and `::store-x` are 555 | different in this respect. 556 | 557 | Otherwise that's really all there is to processors. 558 | 559 | ### Graphs 560 | 561 | The Cues graph DSL matches processor bindings to queues, and connects 562 | everything together into a graph: 563 | 564 | ```clj 565 | (defn example-graph 566 | [db] 567 | {:id ::example 568 | :processors [{:id ::source} 569 | {:id ::inc-x 570 | :in {:input ::source} 571 | :out {:output ::tx}} 572 | {:id ::store-x 573 | :in {:input ::tx} 574 | :opts {:db db}}]}) 575 | ``` 576 | 577 | Both the top level graph and every processor in the `:processors` 578 | catalog requires a unique `:id`. 579 | 580 | By default the processor `:id` is used to dispatch to the 581 | `q/processor` method. However because each `:id` must be unique, you 582 | can also use the `:fn` attribute to dispatch to a different method 583 | from the `:id`: 584 | 585 | ```clj 586 | {:id ::unique-id-1 587 | :fn ::inc-x 588 | ...} 589 | {:id ::unique-id-2 590 | :fn ::inc-x 591 | ...} 592 | ``` 593 | 594 | The keys in the `:in` and `:out` maps are always _processor bindings_ 595 | and the values are always _queue ids_. 596 | 597 | Taking `::inc-x` as an example: 598 | 599 | ```clj 600 | {:id ::inc-x 601 | :in {:input ::source} 602 | :out {:output ::tx}} 603 | ``` 604 | 605 | Here the input messages from the `::source` queue are bound to 606 | `:input`, and output messages bound to `:output` will be placed on the 607 | `::tx` queue. 608 | 609 | Similarly for `::store-x`, input messages from the `::tx` queue are 610 | bound to `:input`: 611 | 612 | ```clj 613 | {:id ::store-x 614 | :in {:input ::tx} 615 | :opts {:db db}} 616 | ``` 617 | 618 | However, notice that there are no output queues defined for 619 | `::store-x`. Instead `::store-x` takes the input messages and 620 | transacts them to the `db` provided via `:opts`: 621 | 622 | ```clj 623 | (defmethod q/processor ::store-x 624 | [{{db :db} :opts} {msg :input}] 625 | (swap! db assoc (:x msg) msg) 626 | nil) 627 | ``` 628 | 629 | If you're familiar with Kafka, `::store-x` would be analogous to a 630 | Sink Connector. Essentially this is an exit node for messages from the 631 | graph. 632 | 633 | You can also define processors similar to Kafka Source 634 | Connectors. This would be the third processor in the catalog: 635 | 636 | ```clj 637 | {:id ::source} 638 | ``` 639 | 640 | A processor with no `:in` or `:out` is considered a source. The `:id` 641 | is also the queue on which some external agent will deposit messages. 642 | 643 | However before we can send messages to the source, we need to 644 | construct and start the graph: 645 | 646 | ```clj 647 | (defonce example-db 648 | (atom nil)) 649 | 650 | (def g 651 | (-> (example-graph example-db) 652 | (q/graph) 653 | (q/start-graph!))) 654 | ``` 655 | 656 | Now we can `send!` our message: 657 | 658 | ```clj 659 | (q/send! g ::source {:x 1}) 660 | ``` 661 | 662 | You can simplify things for users of the graph by setting a default 663 | source: 664 | 665 | ```clj 666 | (defn example-graph 667 | [db] 668 | {:id ::example 669 | :source ::source 670 | :processors [{:id ::source} 671 | ...]}) 672 | ``` 673 | 674 | Now a user of the graph can send messages without having to specify 675 | the source: 676 | 677 | ```clj 678 | (q/send! g {:x 1}) 679 | ``` 680 | 681 | The message will then move through the queues and processors in the 682 | graph until it is deposited in the `example-db` atom by the 683 | `::store-x` sink: 684 | 685 | ```clj 686 | @example-db 687 | ;; => 688 | {2 {:x 2}} ; :x was incremented by ::inc-x 689 | 690 | ;; get all the messages in the graph, each key is a queue 691 | (q/all-graph-messages g) 692 | ;; => 693 | {::source ({:x 1}) 694 | ::tx ({:x 2})} 695 | ``` 696 | 697 | You can stop all the graph processors using: 698 | 699 | ```clj 700 | (q/stop-graph! g) 701 | ``` 702 | 703 | And close the graph and all its associated queues and tailers: 704 | 705 | ```clj 706 | (q/close-graph! g) 707 | ``` 708 | 709 | Finally, we've already seen that we can use the `:opts` attribute to pass 710 | options and resources to _specific_ processors. However, we can pass a 711 | general systems map to _all_ processors in the graph via the `:system` 712 | attribute: 713 | 714 | ```clj 715 | (defn example-graph 716 | [db connection path] 717 | {:id ::example 718 | :system {:connection connection 719 | :path path} 720 | :processors [{:id ::processor 721 | :in {:input ::queue-in} 722 | :out {:output ::queue-out} 723 | :opts {:db db}} 724 | ...]}) 725 | 726 | (defmethod q/processor ::processor 727 | [{{db :db} :opts 728 | {:keys [connection path]} :system} {msg :input}] 729 | ...) 730 | ``` 731 | 732 | ### Topic and Type Filters 733 | 734 | Cues provides an additional layer of control over how your messages 735 | pass through your graph: topic and type filters. 736 | 737 | Here is a processor with a type filter: 738 | 739 | ```clj 740 | {:id ::store-x 741 | :types :q.type/doc 742 | :in {:in ::tx}} 743 | ``` 744 | 745 | This processor will only handle messages whose `:q/type` attribute is 746 | `:q.type/doc`: 747 | 748 | ```clj 749 | ;; yes 750 | {:q/type :q.type/doc 751 | :attr 1 752 | :q/meta {...}} 753 | 754 | ;; no 755 | {:q/type :q.type/control 756 | :control :stop 757 | :q/meta {...}} 758 | 759 | ;; no 760 | {:x 1} 761 | ``` 762 | 763 | Similarly, we can define topic filters: 764 | 765 | ```clj 766 | {:id ::db.query/processor 767 | :topics #{:db.topic/query 768 | :db.topic/control} 769 | :in {:in ::source} 770 | :out {:out ::tx}} 771 | ``` 772 | 773 | This processor will only handle messages whose `:q/topics` map 774 | contains a truthy value for at least one of the topics: 775 | 776 | ```clj 777 | ;; yes 778 | {:q/topics {:db.topic/query [{:some [:domain :query]}]} 779 | :q/meta {...}} 780 | 781 | ;; yes 782 | {:q/topics {:db.topic/control true} 783 | :msg.control/signal :stop 784 | :msg/description "context" 785 | :q/meta {...}} 786 | 787 | ;; no 788 | {:q/topics {:db.topic/write {:doc {:name "doc" :description "description"} 789 | :db.topic/log true} 790 | :q/meta {...}} 791 | 792 | ;; no 793 | {:x 1} 794 | ``` 795 | 796 | The `:q/topic` values in messages can be arbitrary data. 797 | 798 | Of course if you require some other kind of filtering besides what 799 | type and topics provides, you can always pass in your own filters via 800 | processor `:opts`, and perform filtering in the processor function 801 | yourself: 802 | 803 | ```clj 804 | {:id ::processor 805 | :in {:in ::source} 806 | :out {:out ::tx} 807 | :opts {:my-filter :category}} 808 | 809 | (defmethod q/processor ::processor 810 | [{{:keys [my-filter]} :opts} {msg :in}] 811 | (when (predicate? my-filter msg) 812 | ...)) 813 | ``` 814 | 815 | ### Topologies 816 | 817 | There are several variations of processors besides sources and sinks. 818 | 819 | Join processors take messages from multiple queues, blocking until 820 | _all_ queues have a message available. They then write a single 821 | message to one output queue: 822 | 823 | ```clj 824 | {:id ::join-processor 825 | :in {:source ::source 826 | :control ::control} 827 | :out {:output ::tx}} 828 | ``` 829 | 830 | There is an `alts!!` variation of a join processor that will take the 831 | first message available from a set of input queues. 832 | 833 | ```clj 834 | {:id ::alts-processor 835 | :alts {:s1 ::crawler-old 836 | :s2 ::crawler-new 837 | :s3 ::crawler-experimental} 838 | :out {:output ::tx}} 839 | ``` 840 | 841 | Fork processors write messages to multiple queues: 842 | 843 | ```clj 844 | {:id ::fork-processor 845 | :in {:input ::source} 846 | :out {:tx ::tx 847 | :log ::log}} 848 | ``` 849 | 850 | Message delivery on fork output queues is atomic, and exactly once 851 | semantics _will_ apply. However, not all output messages will appear 852 | on their respective output queues at the same time. 853 | 854 | Join/forks or alts/forks both read from multiple input queues and 855 | deliver to multiple output queues: 856 | 857 | ```clj 858 | {:id ::join-fork-processor 859 | :in {:source ::source 860 | :control ::control} 861 | :out {:tx ::tx 862 | :log ::log}} 863 | 864 | {:id ::alt-fork-processor 865 | :alt {:source ::source 866 | :control ::control} 867 | :out {:tx ::tx 868 | :log ::log}} 869 | ``` 870 | 871 | You can conditionally write to different output queues based on which 872 | bindings you return: 873 | 874 | ```clj 875 | (defmethod q/processor ::join-fork-conditional 876 | [process msgs] 877 | (let [n (transduce (map :x) + (vals msgs)) 878 | msg {:x n}] 879 | {:even (when (even? n) msg) 880 | :odd (when (odd? n) msg)})) 881 | ``` 882 | 883 | You can also dynamically compute input and output bindings based on 884 | the processor definition, which is available in the first argument of 885 | the processor under the `:config` attribute: 886 | 887 | ```clj 888 | (defmethod q/processor ::broadcast 889 | ;; Broadcast to all the output bindings of the processor. 890 | [process in-msgs] 891 | (let [out-msgs (compute-out-msgs in-msgs) 892 | out-bindings (keys (:out (:config process)))] 893 | (zipmap out-bindings (repeat out-msgs)))) 894 | ``` 895 | 896 | Once you have connected a set of processors together into a graph, you 897 | can inspect the topology of the graph directly: 898 | 899 | ```clj 900 | (q/topology g) 901 | ;; => 902 | {:nodes #{::store-x ::source ::inc-x} 903 | :deps {:dependencies 904 | {::store-x #{::inc-x} 905 | ::inc-x #{::source}} 906 | :dependents 907 | {::inc-x #{::store-x} 908 | ::source #{::inc-x}}}} 909 | ``` 910 | 911 | And compute properties of the topology: 912 | 913 | ```clj 914 | (require '[cues.deps :as deps]) 915 | 916 | (deps/transitive-dependencies (q/topology g) ::store-x) 917 | ;; => 918 | #{::source ::inc-x} 919 | ``` 920 | 921 | There's a number of topology functions available in 922 | [`cues.deps`](https://github.com/zalky/cues/blob/main/src/cues/deps.clj). This 923 | namespace provides everything that 924 | [`com.stuartsierra/dependency`](https://github.com/stuartsierra/dependency) 925 | does, but extends functionality to support disconnected graphs. 926 | 927 | There's also a couple of helper functions in `cues.util` for merging 928 | and re-binding processor catalogs: 929 | 930 | ```clj 931 | {:processors (-> (util/merge-catalogs cqrs/base-catalog 932 | features-a-catalog 933 | features-b-catalog) 934 | (util/bind-catalog {::my-old-error-queue ::new-error-queue 935 | ::features/tx-queue ::new-tx-queue 936 | ::features/undo-queue ::new-undo-queue}))} 937 | ``` 938 | 939 | This makes graph definitions easier to reuse. 940 | 941 | ## Errors and Exactly Once Message Delivery 942 | 943 | Cues provides persistent, brokerless, and fault tolerant _exactly 944 | once_ message delivery for graph processors, but the approach depends 945 | on whether a processor has side-effects or not. 946 | 947 | For pure processors that do _not_ have side effects, _exactly once_ 948 | delivery semantics work out of the box. You do not have to do 949 | anything. 950 | 951 | For processors that _do_ have side-effects, like sink processors, Cues 952 | provides _at least once_ delivery semantics on the processor, while 953 | also exposing a special delivery hash to user code. 954 | 955 | This delivery hash is unique to the set of input messages being 956 | delivered, and is invariant across multiple delivery 957 | attempts. Processors can then use the hash to implement _idempotency_ 958 | or _exactly once_ delivery semantics for their side effects: 959 | 960 | ```clj 961 | (defmethod q/processor ::sink 962 | [{:keys [delivery-hash] :as process} msgs] 963 | ;; Use delivery-hash to enforce idempotency or exactly once delivery 964 | ;; on side-effects. 965 | ...) 966 | ``` 967 | 968 | For example, consider a sink processor that takes messages from 969 | several input queues, combines them, and writes a result to a 970 | database. If this fails at any point, the processor will attempt to 971 | re-deliver these messages at least once more. 972 | 973 | To ensure the result is written to the database exactly once, the user 974 | code can collocate the delivery hash with the result in the db using 975 | transactional semantics, checking that the transaction has not already 976 | been written. 977 | 978 | There are other approaches for ensuring exactly once delivery on 979 | side-effects, but all of them would leverage the delivery hash. 980 | 981 | ### Code Changes 982 | 983 | For processors with _no_ side-effects, exactly once delivery semantics 984 | still hold even if the user code in the processor has changed between 985 | failure and restart. The new code will be used to complete the 986 | delivery. 987 | 988 | For processors _with_ side-effects, this really depends on the changes 989 | that are made to any user code that leverages the delivery hash. 990 | 991 | However changes to the _topology_ of the graph (the connections 992 | between processors) make exactly once delivery semantics inherently 993 | ambiguous. 994 | 995 | In such cases you might see the following log output on restart: 996 | 997 | ``` 998 | 2023-03-17T19:34:52.085Z user INFO [cues.queue:734] - Topology changed, no snapshot for tailer :some-generated/id 999 | ``` 1000 | 1001 | This is not necessarily an error, just informing you that after the 1002 | topology change the input tailer has lost its recovery context. 1003 | 1004 | ### Message Delivery: Handled versus Unhandled Errors 1005 | 1006 | By default, Cues does not handle any exceptions that bubble up from 1007 | the processor. Instead, they are treated as failed delivery attempts, 1008 | and the processor immediately quits. 1009 | 1010 | On restart, exactly once delivery semantics will ensure that messages 1011 | are not lost and delivery is retried. However this is still not very 1012 | robust and you probably want to handle those exceptions. 1013 | 1014 | While you can always handle errors directly in the processor code, you 1015 | can also configure Cues to handle exceptions for you. Just provide an 1016 | `:errors` queue to either the graph or an individual processor: 1017 | 1018 | ```clj 1019 | (defn graph 1020 | [db] 1021 | {:id ::example 1022 | :errors ::graph-errors 1023 | :processors [{:id ::source} 1024 | {:id ::inc-x 1025 | :in {:input ::source} 1026 | :out {:output ::tx} 1027 | :errors ::inc-x-errors} 1028 | {:id ::store-x 1029 | :in {:input ::tx} 1030 | :opts {:db db}}]}) 1031 | ``` 1032 | 1033 | When configured in this way, any uncaught exceptions that are _not_ of 1034 | type `java.lang.InterruptedException` or `java.lang.Error` that bubble 1035 | up from the processor will be serialized and delivered to the error 1036 | queue. 1037 | 1038 | Error messages are considered "delivered" according to exactly once 1039 | delivery semantics, and the processor will not retry. In other words: 1040 | a message will be delivered exactly once _either_ to an output queue, 1041 | _or_ to an error queue _once_, but never both. 1042 | 1043 | You can then read back errors on the error queues: 1044 | 1045 | ```clj 1046 | (q/graph-messages g ::inc-x-errors) 1047 | ;;=> 1048 | ({:q/type :q.type.err/processor 1049 | :err.proc/config {:id ::inc-x 1050 | :in {:input ::source} 1051 | :out {:output ::tx} 1052 | :errors ::inc-x-errors 1053 | :strategy ::q/exactly-once} 1054 | :err.proc/messages {::source {:x 1}} 1055 | :err/cause {:via [{:type java.lang.Exception 1056 | :message "Oops" 1057 | :at [cues.build$eval77208$fn__77210 invoke "build.clj" 11]}] 1058 | :trace 1059 | [[cues.build$eval77208$fn__77210 invoke "build.clj" 11] 1060 | [cues.queue$wrap_select_processor$fn__75200 invoke "queue.clj" 1061] 1061 | [cues.queue$wrap_imperative$fn__75194 invoke "queue.clj" 1046] 1062 | ... 1063 | [java.util.concurrent.ThreadPoolExecutor runWorker "ThreadPoolExecutor.java" 1128] 1064 | [java.util.concurrent.ThreadPoolExecutor$Worker run "ThreadPoolExecutor.java" 628] 1065 | [java.lang.Thread run "Thread.java" 829]] 1066 | :cause "Oops"}}) 1067 | ``` 1068 | 1069 | By default, Cues creates a generic error message like the one above, 1070 | which contains the processor's configuration, a stacktrace, and the 1071 | input message that triggered the error. 1072 | 1073 | However, you can provide additional context to raised errors with the 1074 | `cues.error/wrap-error` macro: 1075 | 1076 | ```clj 1077 | (require '[cues.errors :as err]) 1078 | 1079 | (defmethod q/processor ::inc-x 1080 | [process {msg :input}] 1081 | (err/wrap-error {:q/type :my-error-type 1082 | :more-context context 1083 | :more-data data} 1084 | ...)) 1085 | ``` 1086 | 1087 | The `wrap-error` macro merges the provided context into the raised 1088 | error, then re-throws it. 1089 | 1090 | Of course, you can always catch and handle errors yourself, and place 1091 | them on arbitrary queues of your choice. Ultimately handled errors are 1092 | just like any other data in the graph. 1093 | 1094 | ### At Most Once Message Delivery 1095 | 1096 | There are [generally three 1097 | strategies](https://medium.com/@madhur25/meaning-of-at-least-once-at-most-once-and-exactly-once-delivery-10e477fafe16) 1098 | that exist for message delivery in systems where failure is a 1099 | possibility: 1100 | 1101 | 1. At most once 1102 | 2. At least once 1103 | 3. Exactly once 1104 | 1105 | Cues provides `::q/exactly-once` message delivery by default, but you 1106 | can optionally configure graphs to use `::q/at-most-once` delivery 1107 | semantics instead: 1108 | 1109 | ```clj 1110 | (require '[cues.queue :as q]) 1111 | 1112 | (defn example-graph 1113 | [db] 1114 | {:id ::example 1115 | :strategy ::q/at-most-once 1116 | :processors [{:id ::source} 1117 | ...]}) 1118 | ``` 1119 | 1120 | With _at most once_ semantics any processor step is only ever 1121 | attempted once, and never retried. While failures may result in 1122 | _dropped messages_, this provides two modest benefits if that is not a 1123 | problem: 1124 | 1125 | 1. Approximately 30-40% faster graph performance 1126 | 2. You can avoid implementing idempotency on side-effects using the 1127 | delivery hash: processor steps are simply never retried 1128 | 1129 | In contrast _at least once_ semantics pose no meaningful benefits with 1130 | respect to _exactly once_ delivery, and so outside of processors with 1131 | side-effects (where it is the default and [explained 1132 | previously](#errors)) that strategy is not provided. 1133 | 1134 | ## Queue Configuration 1135 | 1136 | Whether used as primitives or as part of a graph, the following queue 1137 | properties are configurable: 1138 | 1139 | 1. Path of the queue data on disk 1140 | 2. Message metadata 1141 | 3. Queue data expiration and cycle handlers 1142 | 1143 | For primitives these are passed as an options map to the queue 1144 | constructor: 1145 | 1146 | ```clj 1147 | (def q (q/queue ::queue-id {:queue-path "data/example" 1148 | :queue-meta #{:q/t :q/time} 1149 | :transient true})) 1150 | ``` 1151 | 1152 | The default path for all queues is `"data/queues"` in your project 1153 | root. 1154 | 1155 | The same options can be passed to queues that participate in a graph 1156 | using the queue id: 1157 | 1158 | ```clj 1159 | (defn example-graph 1160 | [db] 1161 | {:id ::example 1162 | :queue-opts {::queue-id {:queue-path "data/example" 1163 | :queue-meta #{:q/t :q/time} 1164 | :transient true} 1165 | ::source {...} 1166 | ::tx {...}} 1167 | :processors [{:id ::source} 1168 | {:id ::inc-x 1169 | :in {:input ::source} 1170 | :out {:output ::tx}} 1171 | ...]}) 1172 | ``` 1173 | 1174 | For graphs, you can specify a set of default queue options. The 1175 | options for specific queues will be merged with the defaults: 1176 | 1177 | ```clj 1178 | (require '[cues.queue :as q]) 1179 | 1180 | (defn example-graph 1181 | [db] 1182 | {:id ::example 1183 | :queue-opts {::q/default {:queue-path "data/default-path" 1184 | :queue-meta #{:q/t}} 1185 | ::source {:queue-path "data/other"} ; merge ::q/default ::source 1186 | ::tx {...}} ; merge ::q/default ::tx 1187 | :processors [{:id ::source} 1188 | {:id ::inc-x 1189 | :in {:input ::source} 1190 | :out {:output ::tx}} 1191 | ...]}) 1192 | ``` 1193 | 1194 | The full set of options are: 1195 | 1196 | 1. **`:queue-path`**: The path on disk where the queue data files are 1197 | stored. It is perfectly fine for multiple queues to share the same 1198 | `:queue-path`. 1199 | 1200 | 2. **`:queue-meta`**: Normally messages are unchanged by the 1201 | implementation when written to disk. However, setting this option 1202 | can ensure three built-in metadata attributes are automatically 1203 | added on write. 1204 | 1205 | The three attributes, `#{:q/t :q/time :tx/t}`, are discussed in the 1206 | next section. But the following shows how the `:q/t` attribute 1207 | could be configured for all queues, and all three attributes for 1208 | the `::tx` queue: 1209 | 1210 | ```clj 1211 | (defn example-graph 1212 | [db] 1213 | {:id ::example 1214 | :queue-opts {::q/default {:queue-path "data/example" 1215 | :queue-meta #{:q/t}} 1216 | ::tx {:queue-meta #{:q/t :q/time :tx/t}}} 1217 | :processors [{:id ::source} 1218 | {:id ::inc-x 1219 | :in {:input ::source} 1220 | :out {:output ::tx}} 1221 | ...]}) 1222 | ``` 1223 | 1224 | You can also set `:queue-meta` to `false`, in which case that queue 1225 | will actively _remove all metadata_ from any message before writing 1226 | it to disk. 1227 | 1228 | 3. **`:transient`**: By default all messages written to disk persist 1229 | forever. Setting `:transient true` will configure the roll-cycle of 1230 | queue data files to be daily, and for data files to be deleted 1231 | after 10 cycles (10 days). With `:transient true` you are only ever 1232 | storing 10 days of data. 1233 | 1234 | 4. All the options supported by [Tape's underlying 1235 | implementation](https://github.com/mpenet/tape/blob/615293e2d9eeaac36b5024f9ca1efc80169ac75c/src/qbits/tape/queue.clj#L26-L45) 1236 | are also configurable. This includes direct control over the roll 1237 | cycle and cycle handlers. For example, if the pre-defined 1238 | `:transient` configuration is not suitable to your needs, you could 1239 | use these settings to define new roll-cycle behaviour. See the 1240 | doc-string in the link for details. 1241 | 1242 | ## Queue Metadata 1243 | 1244 | You can always model, manage and propagate message metadata 1245 | yourself. However Cues provides some built-in metadata functionality 1246 | that should cover many use cases. 1247 | 1248 | There are three metadata attributes `:q/t`, `:q/time`, and 1249 | `:tx/t`. Each of these is added to the body of the message under the 1250 | root `:q/meta` attribute: 1251 | 1252 | ```clj 1253 | (first (q/graph-messages g ::tx)) 1254 | ;; => 1255 | {:x 2 1256 | :q/meta {:q/queue {::source {:q/t 83318070575102 1257 | :q/time #object[java.time.Instant 0x6805b000 "2023-02-11T22:58:26.650462Z"]} 1258 | ::tx {:q/t 83318070575104 1259 | :q/time #object[java.time.Instant 0x40cfaf7b "2023-02-11T22:58:26.655232Z"]}} 1260 | :tx/t 83318070575104}} 1261 | ``` 1262 | 1263 | - **`:q/t`** is the index on the queue at which the message was 1264 | written. This is a _per queue_ metadata attribute. If `:q/t` is 1265 | enabled on multiple queues, the provenance of each message will 1266 | accumulate as it passes from one queue to the next. You would 1267 | typically use `:q/t` for audit or debugging purposes. 1268 | 1269 | - **`:q/time`** is similar, except instead of an index, it collects 1270 | the time instant at which the message was written to that queue. 1271 | 1272 | - **`:tx/t`** is pretty much identical to `:q/t`: it is also derived 1273 | from the index on the queue at which the message was 1274 | written. However unlike `:q/t` the _semantics_ of `:tx/t` are meant 1275 | to be a _global_ transaction t that can be used to achieve 1276 | [serializability](https://en.wikipedia.org/wiki/Serializability) (in 1277 | the transactional sense) of messages anywhere in the graph. 1278 | 1279 | Typically you would enable `:tx/t` on one queue, and use this as 1280 | your source of truth throughout the rest of the graph. However 1281 | achieving serializability also depends to a great extent on the 1282 | topology of the graph, and the nature of the processors. Simply 1283 | enabling `:tx/t` will not by itself be enough to ensure 1284 | serializability. You need to understand how the properties of your 1285 | graph and processors determine serializability. 1286 | 1287 | ### Graph Metadata 1288 | 1289 | Graph processors also have metadata semantics. Processors will 1290 | automatically propagate the merged `:q/meta` from all input messages 1291 | into any output messages on queues that have any metadata attributes 1292 | configured. This means that you do not need to explicitly propagate 1293 | `:q/meta` data in your processor functions. 1294 | 1295 | ## Utility Functions 1296 | 1297 | There some utility functions in `cues.queue` that are worth 1298 | mentioning. 1299 | 1300 | If you already have a tailer, `q/messages` will return a lazy list of 1301 | messages: 1302 | 1303 | ```clj 1304 | (->> (q/messages tailer) 1305 | (map do-something-to-message) 1306 | ...) 1307 | ``` 1308 | 1309 | There are also functions that will process messages eagerly from 1310 | queues. Be careful, these load _all_ the messages on the queue into 1311 | memory: 1312 | 1313 | ```clj 1314 | ;; get all message from queue object 1315 | (q/all-messages queue) 1316 | 1317 | ;; get all messages from a queue in a graph object 1318 | (q/graph-messages graph ::queue-id) 1319 | 1320 | ;; get all messages from a graph object 1321 | (q/all-graph-messages graph) 1322 | ``` 1323 | 1324 | There are also a set of functions for managing queue files. By default 1325 | data file deletion will prompt you to confirm: 1326 | 1327 | ```clj 1328 | ;; Close and delete the queue's data files 1329 | (q/delete-queue! queue) 1330 | 1331 | ;; Close and delete all queues in a graph 1332 | (q/delete-graph-queues! g) 1333 | 1334 | ;; Delete all queues in default queue path, or in the path 1335 | ;; provided. 1336 | (q/delete-all-queues!) 1337 | (q/delete-all-queues "data/example") 1338 | ``` 1339 | 1340 | ## Runway: Developing Stateful Systems 1341 | 1342 | Managing the lifecycle of any stateful application requires care, 1343 | especially in a live coding environment. With processor threads and 1344 | stateful resources running in the background, a Cues graph is no 1345 | different. And while you can certainly manage Cues graphs manually 1346 | from the REPL, it is easier to let a framework to manage your graph's 1347 | lifecycle for you. 1348 | 1349 | This repository demonstrates how you might do this with 1350 | [Runway](https://github.com/zalky/runway), a 1351 | [`com.stuartsierra.component`](https://github.com/stuartsierra/component) 1352 | reloadable build library for managing the lifecycles of complex 1353 | applications. 1354 | 1355 | See the [`deps.edn`](https://github.com/zalky/cues/blob/main/deps.edn) 1356 | file for how to configure Runway aliases, and the 1357 | [`cues.build`](https://github.com/zalky/cues/blob/main/build/cues/build.clj) 1358 | namespace for the `com.stuartsierra.component` example Cues graph. 1359 | 1360 | ## Data Serialization 1361 | 1362 | Data serialization is done via the excellent 1363 | [Nippy](https://github.com/ptaoussanis/nippy) library. It is very 1364 | fast, and you can extend support for custom types and records using 1365 | `nippy/extend-freeze` and `nippy/extend-thaw`. See the Nippy 1366 | documentation for more details. 1367 | 1368 | ## Java 11 & 17 1369 | 1370 | Chronicle Queue [works under both Java 11 and 1371 | 17](https://chronicle.software/chronicle-support-java-17/). However 1372 | some JVM options need to be set: 1373 | 1374 | ```clj 1375 | {:cues/j17 {:jvm-opts ["--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED" 1376 | "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED" 1377 | "--add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED" 1378 | "--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED" 1379 | "--add-opens=jdk.compiler/com.sun.tools.javac=ALL-UNNAMED" 1380 | "--add-opens=java.base/java.lang=ALL-UNNAMED" 1381 | "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED" 1382 | "--add-opens=java.base/java.io=ALL-UNNAMED" 1383 | "--add-opens=java.base/java.util=ALL-UNNAMED"]}} 1384 | ``` 1385 | 1386 | See the Cues 1387 | [`deps.edn`](https://github.com/zalky/cues/blob/main/deps.edn#L30-L38) 1388 | file for what this looks like as an alias. 1389 | 1390 | ## Chronicle Queue Analytics (disabled by default) 1391 | 1392 | Chronicle Queue is a great open source product, but it enables 1393 | [analytics 1394 | collection](https://github.com/OpenHFT/Chronicle-Map/blob/ea/DISCLAIMER.adoc) 1395 | by default (opt-out) to improve its product. 1396 | 1397 | Cues changes this by **_removing and disabling_ analytics by 1398 | default**, making it opt-in. 1399 | 1400 | If for some reason you want analytics enabled, you can add the 1401 | following into your `deps.edn` file: 1402 | 1403 | ```clj 1404 | net.openhft/chronicle-analytics {:mvn/version "2.24ea0"} 1405 | ``` 1406 | 1407 | When enabled, the analytics engine will emit a message the first time 1408 | software is run, and generate a `~/.chronicle.analytics.client.id` 1409 | file in the user's home directory. 1410 | 1411 | ## Getting Help 1412 | 1413 | First you probably want to check out the 1414 | [Chronicle Queue](https://github.com/OpenHFT/Chronicle-Queue) 1415 | documentation, as well as their 1416 | [FAQ](https://github.com/OpenHFT/Chronicle-Queue/blob/ea/docs/FAQ.adoc). 1417 | 1418 | Otherwise you can either submit an issue here on Github, or tag me 1419 | (`@zalky`) with your question in the `#clojure` channel on the 1420 | [Clojurians](https://clojurians.slack.com) slack. 1421 | 1422 | ## License 1423 | 1424 | Cues is distributed under the terms of the Apache License 2.0. 1425 | -------------------------------------------------------------------------------- /build/cues/build.clj: -------------------------------------------------------------------------------- 1 | (ns cues.build 2 | (:require [com.stuartsierra.component :as component] 3 | [cues.queue :as q] 4 | [runway.core :as run] 5 | 6 | ;; Loaded for use 7 | [cues.log])) 8 | 9 | (defmethod q/processor ::inc-x 10 | [_ {msg :input}] 11 | {:output (update msg :x inc)}) 12 | 13 | (defmethod q/processor ::store-x 14 | [{{db :db} :opts} {msg :input}] 15 | (swap! db assoc (:x msg) (dissoc msg :q/meta))) 16 | 17 | (defn graph-spec 18 | [db] 19 | {:id ::example 20 | :source ::source 21 | :errors ::errors 22 | :queue-opts {::tx {:queue-meta #{:tx/t}}} 23 | :processors [{:id ::source} 24 | {:id ::inc-x 25 | :in {:input ::source} 26 | :out {:output ::tx}} 27 | {:id ::store-x 28 | :in {:input ::tx} 29 | :opts {:db db}}]}) 30 | 31 | (defonce db 32 | (atom {})) 33 | 34 | (defrecord GraphExample [] 35 | component/Lifecycle 36 | (start [component] 37 | (->> (graph-spec db) 38 | (q/graph) 39 | (q/start-graph!) 40 | (merge component))) 41 | 42 | (stop [component] 43 | (->> component 44 | (q/stop-graph!) 45 | (q/close-graph!)) 46 | (GraphExample.))) 47 | 48 | (def components 49 | {:graph [->GraphExample]}) 50 | 51 | (def dependencies 52 | {}) 53 | 54 | (defn test-system 55 | [] 56 | (run/assemble-system components dependencies)) 57 | -------------------------------------------------------------------------------- /build/cues/dev.clj: -------------------------------------------------------------------------------- 1 | (ns cues.dev) 2 | 3 | (in-ns 'user) 4 | 5 | (require '[clojure.java.io :as io] 6 | '[criterium.core :as b] 7 | '[cues.build :as build] 8 | '[cues.deps :as deps] 9 | '[cues.repl :as repl] 10 | '[cues.queue :as q] 11 | '[qbits.tape.appender :as app] 12 | '[qbits.tape.queue :as queue] 13 | '[qbits.tape.tailer :as tail] 14 | '[runway.core :as run] 15 | '[taoensso.timbre :as log]) 16 | 17 | (set! *warn-on-reflection* true) 18 | -------------------------------------------------------------------------------- /build/cues/repl.clj: -------------------------------------------------------------------------------- 1 | (ns cues.repl 2 | (:require [cues.build :as build] 3 | [cues.deps :as deps] 4 | [cues.queue :as q] 5 | [runway.core :as run] 6 | [taoensso.timbre :as log])) 7 | 8 | (defn system 9 | "Returns the full system, or specific dependencies." 10 | ([] 11 | run/system) 12 | ([dependencies] 13 | (select-keys (system) dependencies))) 14 | 15 | (defn graph 16 | [] 17 | (:graph (system))) 18 | 19 | (defn restart! 20 | "Restarts system without spamming repl." 21 | [] 22 | (run/restart) 23 | nil) 24 | 25 | (defn stop! 26 | "Stop system without spamming repl." 27 | [] 28 | (run/stop) 29 | nil) 30 | 31 | (defn start! 32 | "Start system without spamming repl." 33 | [] 34 | (run/start) 35 | nil) 36 | 37 | (defn delete-graph-and-restart! 38 | [] 39 | (let [g (graph)] 40 | (stop!) 41 | (log/info "Deleting system graph and restarting...") 42 | (q/close-and-delete-graph! g true) 43 | (start!)) 44 | nil) 45 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:deps {com.stuartsierra/dependency {:mvn/version "1.0.0"} 2 | com.taoensso/nippy {:mvn/version "3.2.0"} 3 | com.taoensso/timbre {:mvn/version "5.2.1"} 4 | expound/expound {:mvn/version "0.9.0"} 5 | io.zalky/cinch {:mvn/version "0.1.1"} 6 | io.zalky/tape {:mvn/version "0.3.5-rc2" :exclusions [net.openhft/chronicle-analytics]} 7 | net.openhft/chronicle-queue {:mvn/version "5.24ea9" :exclusions [net.openhft/chronicle-analytics]} 8 | org.clojure/clojure {:mvn/version "1.11.1"}} 9 | :paths ["src"] 10 | :aliases {:repl {:extra-paths ["build"] 11 | :extra-deps {cider/cider-nrepl {:mvn/version "0.28.5"} 12 | criterium/criterium {:mvn/version "0.4.4"} 13 | io.zalky/runway {:mvn/version "0.2.2" :exclusions [com.taoensso/encore]} 14 | nrepl/nrepl {:mvn/version "0.8.3"} 15 | refactor-nrepl/refactor-nrepl {:mvn/version "3.6.0"}} 16 | :exec-fn runway.core/exec 17 | :exec-args {runway.core/watcher {} 18 | runway.nrepl/server {} 19 | cues.dev true}} 20 | :test {:extra-paths ["test"] 21 | :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git" 22 | :sha "705ad25bbf0228b1c38d0244a36001c2987d7337"} 23 | org.slf4j/slf4j-nop {:mvn/version "2.0.6"}} 24 | :jvm-opts ["-Dprofile=test"] 25 | :main-opts ["-m" "cognitect.test-runner"]} 26 | :server {:extra-paths ["build"] 27 | :extra-deps {io.zalky/runway {:mvn/version "0.2.2" :exclusions [com.taoensso/encore]} 28 | org.slf4j/slf4j-nop {:mvn/version "2.0.6"}} 29 | :exec-fn runway.core/exec 30 | :exec-args {runway.core/go {:system cues.build/test-system}} 31 | :jvm-opts ["-XX:-OmitStackTraceInFastThrow"]} 32 | :cues/j17 {:jvm-opts ["--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED" 33 | "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED" 34 | "--add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED" 35 | "--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED" 36 | "--add-opens=jdk.compiler/com.sun.tools.javac=ALL-UNNAMED" 37 | "--add-opens=java.base/java.lang=ALL-UNNAMED" 38 | "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED" 39 | "--add-opens=java.base/java.io=ALL-UNNAMED" 40 | "--add-opens=java.base/java.util=ALL-UNNAMED"]} 41 | :build {:deps {io.zalky/build-clj {:git/url "https://github.com/zalky/build-clj.git" 42 | :git/sha "c8782e887381160f6d34d48425dc2d3a2a40f4cb"}} 43 | :ns-default io.zalky.build}}} 44 | -------------------------------------------------------------------------------- /doc/Messaging-Semantics.md: -------------------------------------------------------------------------------- 1 | 2 | # Messaging Semantics 3 | 4 | There are generally three strategies that exsist for graph processor 5 | message delivery: 6 | 7 | 1. At most once 8 | 2. At least once 9 | 3. Exactly once 10 | 11 | Currently Cues only implements _exactly once_ semantics for graph 12 | processors. This is significantly harder than the other two, but more 13 | useful overall. The main advantage of the other two is performance, 14 | but not by a large margin. 15 | 16 | Nevertheless, _at most once_ and _at least once_ are next in the 17 | feature backlog and will be included in a future release. 18 | 19 | ## Message Delivery 20 | 21 | Input messages for a processor step are considered delivered if either 22 | an output message is successfully written to an output queue, or an 23 | exception is caught and handled. A _handled_ exception means that it 24 | is: 25 | 26 | 1. Logged 27 | 2. Written to an `:error-queue` if the queue has been configured 28 | 29 | An _unhandled_ error will result in a processor interrupt, and 30 | undelivered messages. Once the processor restarts, exactly once 31 | semantics will ensure that the messages are not lost and the delivery 32 | will be retried. It also ensures that once the messages are delivered, 33 | they will not be delivered again. 34 | 35 | ## Exactly Once Algorithm 36 | 37 | The exactly once algorithm is different depending on whether the 38 | processor function has side effects or not. Here, "processor function" 39 | refers to the user code defined by `cues.queue/processor`. Obviously 40 | the Cues implementation will have side effects in order to persist 41 | messages to queues. 42 | 43 | ### Processors Without Side Effects 44 | 45 | The algorithm described below is specifically for a "join" processor, 46 | which accepts multiple input messages and produces a single output 47 | message. However every other kind of processor can generalize from 48 | this one case. 49 | 50 | A transient "backing queue" is needed to implement the algorithm. This 51 | queue is automatically created and isolated from the user. A user 52 | specified error queue can also be configured for delivery of _handled_ 53 | error messages. 54 | 55 | The algorithm can be broken down into two components. 56 | 57 | 1. A persistence component that runs on every processor step 58 | 2. A recovery component that runs one time on processor start 59 | 60 | Each of the components can safely fail at any point. The recovery 61 | component is idempotent. The persistence component is not: if it 62 | fails, the processor must exit and recovery must be run. 63 | 64 | #### The persistence component: runs on every step 65 | 66 | 1. Collect the current index for all input tailers as well as the 67 | processor id into a snapshot map: 68 | 69 | ```clj 70 | {:q/type :q.type/snapshot 71 | :q/proc-id :cues.build/example-processor 72 | :q/tailer-indices {::t1 080928480 73 | ::t2 080928481}} 74 | ``` 75 | 76 | 2. _If and only if this is the first attempt to deliver these 77 | messages_, then write this snapshot map to a backing queue. 78 | 79 | 3. Compute a hash of the snapshot map. This is the delivery hash. The 80 | value is unique to the combination of indices and processor id, and 81 | will evaluate to be the same for each delivery attempt. Because the 82 | messages on the input queues are immutable, their indices and the 83 | processor id is all you need to compute the hash. You do not need 84 | the actual input messages. 85 | 86 | 4. Now read one or more messages from the input tailers, advancing the 87 | relevant tailers. _If this is an alts processor and we are retrying 88 | a delivery attempt_, make sure to read from the same tailer that 89 | was used for the initial attempt regardless of which tailers may 90 | now have messages (4a of the persistence component and 1b of the 91 | recovery component helps you do this). 92 | 93 | a) _Immediately after the read, and if this is an alts processor, 94 | and if this is the first attempt at delivery_, create and alts 95 | snapshot map that contains the id of the alts tailer that was 96 | just read from. Because the availability of messages on an alts 97 | read is non-deterministic, we need to store which tailer was 98 | initially read from for subsequent alts attempts. 99 | 100 | ```clj 101 | {:q/type :q.type/snapshot-alts 102 | :q/tailer-id :some-tailer-id} 103 | ``` 104 | 105 | Write this map to the backing queue. 106 | 107 | 5. Run the processor function, which may or may not return an output 108 | message to be written to the output queue. 109 | 110 | a) If there is an output message, proceed to step 7. 111 | 112 | b) If there is _no_ output message, proceed to step 8. 113 | 114 | 6. If at any point from steps 3 onward an exception is caught and 115 | _handled_: 116 | 117 | a) Proceed to step 7 if the `:error-queue` has been configured to 118 | write an error message. 119 | 120 | b) Proceed to step 8 if _no_ `:error-queue` has been configured, 121 | and no error message should be written. 122 | 123 | 7. Open a transaction to write the output message to the output queue 124 | or an error message to the error queue: 125 | 126 | a) Add the delivery hash to the metadata of the output or error 127 | message: 128 | 129 | ```clj 130 | {:q/type :q.type/some-output-message-type 131 | :q/meta {:q/hash -499796012 ....} 132 | ...} 133 | ``` 134 | 135 | b) Create an attempt map. The type of the attempt map should depend 136 | on whether the message is an output or error message. For 137 | example: 138 | 139 | - Output attempt: `:q.type/attempt-output` 140 | - Error attempt: `:q.type/attempt-error` 141 | 142 | c) Get the index of the provisional write on the output or error 143 | queue, and add this message index to the attempt map: 144 | 145 | ```clj 146 | {:q/type :q.type/attempt-output 147 | :q/message-index 080928480} 148 | ``` 149 | 150 | d) Write the attempt map to the backing queue. 151 | 152 | e) Complete the output or error transaction. The messages are 153 | considered delivered. 154 | 155 | 8. If there is no output or error message to be written (skip this if 156 | you have already done 7): 157 | 158 | a) Create an attempt map of type `:q.type/attempt-nil`. 159 | 160 | b) Add the delivery hash to the attempt map: 161 | 162 | ```clj 163 | {:q/type :q.type/attempt-nil 164 | :q/hash -499796012} 165 | ``` 166 | 167 | c) Write the attempt map to the backing queue. The messages are 168 | considered delivered. 169 | 170 | 9. Continue processing the next set of messages. If at any point 171 | during steps 1-8 an _unhandled_ error occurs, the processor should 172 | immediately interrupt and exit. 173 | 174 | #### The recovery component: runs once on start 175 | 176 | 1. Read the most recent message on the backing queue. Depending on the 177 | message type: 178 | 179 | a) No message on the backing queue: the processor has never made an 180 | attempt, proceed to 4. 181 | 182 | b) `:q.type/snapshot`: The previous attempt failed. Configure the 183 | processor to skip making new snapshots until a delivery attempt 184 | is successful. Reset the tailers to the indices in the snapshot 185 | and proceed to 4. 186 | 187 | c) `:q.type/snapshot-alts`: The previous attempt failed. Configure 188 | the alts processor with the tailer id in the message: all 189 | subsequent alts retries should read from this tailer. Continue 190 | reading backwards on the backing queue until you reach the first 191 | snapshot. Proceed to 1b. 192 | 193 | d) `:q.type/attempt-output`: take the `:q/message-index` from the 194 | attempt map and read back the message at this index on the 195 | _output queue_. Get the delivery hash from the output message 196 | metadata. Proceed to 2. 197 | 198 | e) `:q.type/attempt-error`: take the `:q/message-index` from the 199 | attempt map and read back the message at this index on the 200 | _error queue_. Get the delivery hash from the error message 201 | metadata. Proceed to 2. 202 | 203 | f) `:q.type/attempt-nil`: get the delivery hash directly from the 204 | attempt map. Proceed to 2. 205 | 206 | 2. Read backwards from the most recent message on the backing queue 207 | until you reach the first snapshot. Compute the hash of the 208 | snapshot. 209 | 210 | 3. Then: 211 | 212 | a) If the delivery hash from step 1 and the snapshot hash from step 213 | 2 _are the same_: then the last delivery attempt was successful. 214 | Proceed to 4. 215 | 216 | b) Else, they are _not the same_: the last delivery attempt 217 | failed. Reset tailers to the recovery indices in the snapshot 218 | message. Then proceed to 4. 219 | 220 | 4. Continue processing messages. 221 | 222 | ### Processors With Side Effects 223 | 224 | The exactly once algorithm described above relies on transactionally 225 | collocating the delivery hash with the output message on the output 226 | queue (or the error message on the error queue). 227 | 228 | This is easy enough when there are no side effects and the data model 229 | of the messages and the implementation of the queues are both 230 | known. However, when processors perform side-effects on arbitrary 231 | external resources, the algorithm fails to generalize. 232 | 233 | For these processors, Cues takes an approach similar to Kafka: it 234 | provides _at least once_ messaging semantics on the processor, but 235 | then exposes the delivery hash to user code so that it can use the 236 | hash to implement _idempotency_ or _exactly once_ delivery semantics 237 | for the side effects: 238 | 239 | ```clj 240 | (defmethod q/processor ::sink 241 | [{:keys [delivery-hash] :as process} msgs] 242 | ;; Use delivery-hash to enable idempotence or enforce 243 | ;; exactly once semantics 244 | ...) 245 | ``` 246 | 247 | For example, consider a sink processor that takes messages from some 248 | input queues, combines them, and writes a result to a database. After 249 | an _unhandled_ error, interrupt, and restart, the processor will 250 | attempt to re-deliver these messages at least once. However on each 251 | attempt, the delivery hash will be the same. 252 | 253 | Therefore when writing the result to the database, the user code can 254 | collocate the hash with the result in the db to ensure the result is 255 | only written once. This usually requires some form of transactional 256 | semantics on the target db, or alternatively, some kind of 257 | single-writer configuration. 258 | -------------------------------------------------------------------------------- /src/cues/controllers.clj: -------------------------------------------------------------------------------- 1 | (ns cues.controllers 2 | "Provides a global controller cache. 3 | 4 | Controllers ensure that blocking tailers will unblock when an 5 | appender writes to a blocking queue. Each queue has one and only one 6 | corresponding controller. Once a controller has been bound to a 7 | queue on disk, it should never be removed for the lifetime of the 8 | app, unless the queue has been deleted on disk, in which case, the 9 | controller must be purged from the cache." 10 | (:require [cues.util :as util]) 11 | (:import java.nio.file.Paths)) 12 | 13 | (def ^:private cache 14 | (atom {})) 15 | 16 | (defn- components 17 | [path] 18 | (->> (into-array String []) 19 | (Paths/get path) 20 | (.iterator) 21 | (iterator-seq) 22 | (map str))) 23 | 24 | (defn lookup 25 | [path xtor] 26 | (let [c (components path)] 27 | (letfn [(f [cache] 28 | (if-not (get-in cache c) 29 | (assoc-in cache c (xtor)) 30 | cache))] 31 | (get-in (swap! cache f) c)))) 32 | 33 | (defn purge 34 | [path] 35 | (->> path 36 | (components) 37 | (swap! cache util/dissoc-in))) 38 | -------------------------------------------------------------------------------- /src/cues/deps.clj: -------------------------------------------------------------------------------- 1 | (ns cues.deps 2 | "Extends com.stuartsierra.dependency for disconnected graphs." 3 | (:require [cinch.core :as util] 4 | [cinch.spec :as s*] 5 | [clojure.set :as set] 6 | [clojure.spec.alpha :as s] 7 | [com.stuartsierra.dependency :as dep])) 8 | 9 | (s/def ::step 10 | (s*/non-conformer 11 | (s/or :k qualified-keyword? 12 | :s string?))) 13 | 14 | (s/def ::steps 15 | (s/or :dag ::dag 16 | :pipe ::pipeline 17 | :steps ::set-of-steps 18 | :step ::step)) 19 | 20 | (s/def ::set-of-steps 21 | (s*/conform-to 22 | (s/coll-of (s/nilable ::steps) :kind set?) 23 | (partial remove nil?))) 24 | 25 | (s/def ::dag 26 | (s/map-of ::steps ::steps :conform-keys true)) 27 | 28 | (s/def ::pipeline 29 | (s*/conform-to 30 | (s/coll-of (s/nilable ::steps) :kind vector?) 31 | (partial remove nil?))) 32 | 33 | (defn no-dependents? 34 | [g node] 35 | (empty? (dep/immediate-dependents g node))) 36 | 37 | (defn no-dependencies? 38 | [g node] 39 | (empty? (dep/immediate-dependencies g node))) 40 | 41 | (defn out-nodes 42 | [g] 43 | (->> g 44 | (dep/nodes) 45 | (filter (partial no-dependents? g)) 46 | (set))) 47 | 48 | (defn in-nodes 49 | [g] 50 | (->> g 51 | (dep/nodes) 52 | (filter (partial no-dependencies? g)) 53 | (set))) 54 | 55 | (defprotocol DisconnectedDependencyGraphUpdate 56 | (add-node [g node] 57 | "Adds a disconnected node to the dependency graph.")) 58 | 59 | (defrecord DisconnectedDependencyGraph [nodes deps] 60 | dep/DependencyGraph 61 | (immediate-dependencies [_ node] 62 | (dep/immediate-dependencies deps node)) 63 | 64 | (immediate-dependents [_ node] 65 | (dep/immediate-dependents deps node)) 66 | 67 | (transitive-dependencies [_ node] 68 | (dep/transitive-dependencies deps node)) 69 | 70 | (transitive-dependencies-set [_ node-set] 71 | (dep/transitive-dependencies-set deps node-set)) 72 | 73 | (transitive-dependents [_ node] 74 | (dep/transitive-dependents deps node)) 75 | 76 | (transitive-dependents-set [_ node-set] 77 | (dep/transitive-dependents-set deps node-set)) 78 | 79 | (nodes [g] 80 | (:nodes g)) 81 | 82 | dep/DependencyGraphUpdate 83 | (depend [g node dep] 84 | (-> g 85 | (update :deps dep/depend node dep) 86 | (update :nodes util/conjs node dep))) 87 | 88 | (remove-edge [g node dep] 89 | (update g :deps dep/remove-edge node dep)) 90 | 91 | (remove-all [g node] 92 | (-> g 93 | (update :deps dep/remove-all node) 94 | (update :nodes disj node))) 95 | 96 | (remove-node [g node] 97 | (update g :deps dep/remove-node node)) 98 | 99 | DisconnectedDependencyGraphUpdate 100 | (add-node [g node] 101 | (update g :nodes util/conjs node))) 102 | 103 | (defn disconnected-graph 104 | ([] 105 | (disconnected-graph #{})) 106 | ([nodes] 107 | (disconnected-graph nodes (dep/graph))) 108 | ([nodes deps] 109 | (DisconnectedDependencyGraph. nodes deps))) 110 | 111 | (defn merge-graphs 112 | ([] (disconnected-graph)) 113 | ([g] g) 114 | ([{n1 :nodes d1 :deps} 115 | {n2 :nodes d2 :deps}] 116 | (->> (merge-with (partial merge-with set/union) d1 d2) 117 | (disconnected-graph (set/union n1 n2))))) 118 | 119 | (defn depend-graphs 120 | [{n1 :nodes :as g1} 121 | {n2 :nodes :as g2}] 122 | (let [in (in-nodes g1) 123 | out (out-nodes g2) 124 | deps (for [o out 125 | i in] 126 | [i o])] 127 | (reduce 128 | (fn [g [i o]] (dep/depend g i o)) 129 | (merge-graphs g1 g2) 130 | deps))) 131 | 132 | (declare graph*) 133 | 134 | (defn dag-graph 135 | [form] 136 | (reduce-kv 137 | (fn [g form-1 form-2] 138 | (let [g1 (graph* form-1) 139 | g2 (graph* form-2)] 140 | (->> (depend-graphs g2 g1) 141 | (merge-graphs g)))) 142 | (disconnected-graph) 143 | form)) 144 | 145 | (defn pipe-graph 146 | [form] 147 | (loop [g (disconnected-graph) 148 | [g-next & more] (->> form 149 | (remove nil?) 150 | (map graph*))] 151 | (if g-next 152 | (-> g-next 153 | (depend-graphs g) 154 | (recur more)) 155 | g))) 156 | 157 | (defn steps-graph 158 | [form] 159 | (->> form 160 | (remove nil?) 161 | (map graph*) 162 | (reduce merge-graphs))) 163 | 164 | (defn step-graph 165 | [form] 166 | (disconnected-graph #{form})) 167 | 168 | (defn- graph* 169 | [form] 170 | (let [[t subform] form] 171 | (case t 172 | :dag (dag-graph subform) 173 | :pipe (pipe-graph subform) 174 | :step (step-graph subform) 175 | :steps (steps-graph subform)))) 176 | 177 | (defn graph 178 | [expr] 179 | (graph* (s*/parse ::steps expr))) 180 | 181 | (defn transitive-dependencies+ 182 | [graph node] 183 | (-> graph 184 | (dep/transitive-dependencies node) 185 | (conj node))) 186 | 187 | (defn transitive-dependents+ 188 | [graph node] 189 | (-> graph 190 | (dep/transitive-dependents node) 191 | (conj node))) 192 | 193 | (defn two-hops 194 | [deps node] 195 | (->> node 196 | (dep/immediate-dependencies deps) 197 | (mapcat #(dep/immediate-dependencies deps %)) 198 | (not-empty) 199 | (set))) 200 | 201 | ;; Re-bind com.stuartsierra.dependency API 202 | 203 | (def immediate-dependencies dep/immediate-dependencies) 204 | (def immediate-dependents dep/immediate-dependents) 205 | (def transitive-dependencies dep/transitive-dependencies) 206 | (def transitive-dependencies-set dep/transitive-dependencies-set) 207 | (def transitive-dependents dep/transitive-dependents) 208 | (def transitive-dependents-set dep/transitive-dependents-set ) 209 | (def nodes dep/nodes) 210 | 211 | (def depend dep/depend) 212 | (def remove-edge dep/remove-edge) 213 | (def remove-all dep/remove-all) 214 | (def remove-node dep/remove-node) 215 | 216 | (def depends? dep/depends?) 217 | (def dependent? dep/dependent?) 218 | (def topo-sort dep/topo-sort) 219 | (def topo-comparator dep/topo-comparator) 220 | -------------------------------------------------------------------------------- /src/cues/error.clj: -------------------------------------------------------------------------------- 1 | (ns cues.error 2 | (:require [cinch.core :as util])) 3 | 4 | (defn- error-message 5 | [context e] 6 | (or (:err/message context) 7 | (ex-message e))) 8 | 9 | (defn- error-cause 10 | [e] 11 | (let [data (ex-data e)] 12 | (if (:q/type data) 13 | (ex-cause e) 14 | e))) 15 | 16 | (defn- error-data 17 | [context e] 18 | (let [data (ex-data e)] 19 | (if (:q/type data) 20 | (merge context data) 21 | context))) 22 | 23 | (defn error 24 | "Wraps any kind of exception in the Cues error data model, merging 25 | additional context in a transparent way. The intent is to be able to 26 | successively merge in Cues error data as some processor error 27 | bubbles up the implementation stack. Error context is merged, rather 28 | chained. 29 | 30 | Given an exception that is raised in user code: 31 | 32 | exception cause 33 | domain exception e - 34 | cues level 1 (merge c1) e 35 | cues level 2 (merge c1 c2) e 36 | cues level 3 (merge c1 c2 c3) e 37 | ..." 38 | ([e] 39 | (error nil e)) 40 | ([context e] 41 | (let [cause (error-cause e) 42 | m (Throwable->map cause) 43 | data (error-data context e)] 44 | (ex-info (error-message context e) 45 | (merge {:q/type :q.type.err/error 46 | :err/cause m} 47 | data) 48 | cause)))) 49 | 50 | (defmacro wrap-error 51 | [context & body] 52 | `(try 53 | (do ~@body) 54 | (catch InterruptedException e# 55 | (throw e#)) 56 | (catch Exception e# 57 | (throw (error ~context e#))))) 58 | 59 | -------------------------------------------------------------------------------- /src/cues/log.clj: -------------------------------------------------------------------------------- 1 | (ns cues.log 2 | (:require [clojure.spec.alpha :as s] 3 | [taoensso.encore :as enc] 4 | [taoensso.timbre :as log] 5 | [taoensso.timbre.appenders.core :as appenders])) 6 | 7 | (defn std-appender 8 | "Basic appender that appends to stdout and stderr." 9 | [] 10 | (let [f (:fn (appenders/println-appender))] 11 | {:enabled? true 12 | :async? true 13 | :min-level nil 14 | :rate-limit nil 15 | :output-fn :inherit 16 | :fn (fn [data] 17 | (binding [*print-namespace-maps* false] 18 | (log/with-default-outs (f data))))})) 19 | 20 | (defn realise-lazy-seqs 21 | "prints lazy sequences in timbre's log output 22 | https://github.com/ptaoussanis/timbre/pull/200#issuecomment-259535414" 23 | [data] 24 | (letfn [(->str [x] 25 | (if (enc/lazy-seq? x) 26 | (pr-str x) 27 | x))] 28 | (update data :vargs (partial mapv ->str)))) 29 | 30 | (def config 31 | "Basic timbre config." 32 | {:level :debug 33 | :configured "cues" 34 | :ns-whitelist ["user" "runway.*" "cues.*"] 35 | :middleware [realise-lazy-seqs] 36 | :appenders {:std-appender (std-appender)}}) 37 | 38 | ;; Set logging if not already set, but override known dependency 39 | ;; configurations. 40 | (let [c (:configured log/*config*)] 41 | (when (or (nil? c) (contains? #{"cues" "runway"} c)) 42 | (log/set-config! config))) 43 | 44 | (s/check-asserts true) 45 | -------------------------------------------------------------------------------- /src/cues/queue.clj: -------------------------------------------------------------------------------- 1 | (ns cues.queue 2 | "Core Cues API." 3 | (:refer-clojure :exclude [read peek]) 4 | (:require [cinch.core :as util] 5 | [cinch.spec :as s*] 6 | [clojure.java.io :as io] 7 | [clojure.set :as set] 8 | [clojure.spec.alpha :as s] 9 | [clojure.string :as str] 10 | [cues.controllers :as controllers] 11 | [cues.deps :as deps] 12 | [cues.error :as err] 13 | [cues.util :as cutil] 14 | [qbits.tape.appender :as app] 15 | [qbits.tape.codec :as codec] 16 | [qbits.tape.queue :as queue] 17 | [qbits.tape.tailer :as tail] 18 | [taoensso.nippy :as nippy] 19 | [taoensso.timbre :as log]) 20 | (:import clojure.lang.IRef 21 | java.io.File 22 | java.nio.ByteBuffer 23 | java.time.Instant 24 | java.util.stream.Collectors 25 | net.openhft.chronicle.bytes.Bytes 26 | net.openhft.chronicle.queue.ExcerptAppender 27 | [net.openhft.chronicle.queue.impl.single SingleChronicleQueue StoreTailer] 28 | net.openhft.chronicle.queue.util.FileUtil 29 | net.openhft.chronicle.wire.DocumentContext)) 30 | 31 | (def queue-path-default 32 | "data/queues/") 33 | 34 | (defn queue? 35 | [x] 36 | (= (:type x) ::queue)) 37 | 38 | (defn tailer? 39 | [x] 40 | (= (:type x) ::tailer)) 41 | 42 | (defn appender? 43 | [x] 44 | (= (:type x) ::appender)) 45 | 46 | (defn- queue-any? 47 | [x] 48 | (if (or (sequential? x) (set? x)) 49 | (every? queue? x) 50 | (queue? x))) 51 | 52 | (defn id->str 53 | [id] 54 | {:pre [(keyword? id)]} 55 | (if (qualified-keyword? id) 56 | (str (namespace id) "/" (name id)) 57 | (name id))) 58 | 59 | (defn- absolute-path? 60 | [x] 61 | (and (string? x) 62 | (.isAbsolute (io/file x)))) 63 | 64 | (defn- queue-path 65 | [{id :id 66 | {p :queue-path} :opts}] 67 | {:pre [(not (absolute-path? p)) 68 | (not (absolute-path? id)) 69 | (not (absolute-path? queue-path-default))]} 70 | (-> (or p queue-path-default) 71 | (io/file (id->str id)) 72 | (str))) 73 | 74 | (defn- codec 75 | [] 76 | (reify 77 | codec/ICodec 78 | (write [_ x] 79 | (->> x 80 | (nippy/freeze) 81 | (ByteBuffer/wrap))) 82 | (read [_ x] 83 | (->> x 84 | (.array) 85 | (nippy/thaw))))) 86 | 87 | (defn- suffix-id 88 | [id suffix] 89 | {:pre [(keyword? id)]} 90 | (keyword (namespace id) 91 | (str (name id) suffix))) 92 | 93 | (defn- combined-id 94 | [id1 id2] 95 | {:pre [(keyword? id1) 96 | (keyword? id2)]} 97 | (let [ns1 (namespace id1) 98 | ns2 (namespace id2) 99 | n1 (name id1) 100 | n2 (name id2)] 101 | (keyword (->> [ns1 n1 ns2] 102 | (remove nil?) 103 | (interpose ".") 104 | (apply str) 105 | (not-empty)) 106 | n2))) 107 | 108 | (declare last-index) 109 | 110 | (def transient-queue-opts 111 | "Transient queues are purged after 10 days." 112 | {:roll-cycle :small-daily 113 | :cycle-release-tasks [{:type :delete 114 | :after-cycles 10}]}) 115 | 116 | (defn- queue-opts 117 | [{:keys [transient] :as opts}] 118 | (cond-> (-> opts 119 | (assoc :codec (codec)) 120 | (dissoc :transient)) 121 | (true? transient) (merge transient-queue-opts))) 122 | 123 | (defn queue 124 | "Creates a persistent blocking queue. 125 | 126 | Options include: 127 | 128 | :transient 129 | A Boolean. If true, queue backing file will be rolled 130 | daily, and deleted after 10 days. You can configure 131 | alternative behaviour using the :roll-cycle, 132 | :cycle-release-tasks and :cycle-acquire-tasks options 133 | described below. 134 | 135 | The following options are passed along to the underlying 136 | implementation: 137 | 138 | :roll-cycle 139 | How frequently the queue data file on disk is rolled 140 | over. Default is :small-daily. Can be: 141 | 142 | :minutely, :daily, :test4-daily, :test-hourly, :hourly, 143 | :test-secondly, :huge-daily-xsparse, :test-daily, 144 | :large-hourly-xsparse, :large-daily, :test2-daily, 145 | :xlarge-daily, :huge-daily, :large-hourly, :small-daily, 146 | :large-hourly-sparse 147 | 148 | :autoclose-on-jvm-exit? 149 | A Boolean. Whether to cleanly close the JVM on 150 | exit. Default is true. 151 | 152 | :cycle-release-tasks 153 | Tasks to run on queue cycle release. For more, see 154 | qbits.tape.cycle-listener in the Tape library. 155 | 156 | :cycle-acquire-tasks 157 | Tasks to run on queue cycle acquisition. For more, see 158 | qbits.tape.cycle-listener in the Tape library." 159 | ([id] 160 | (queue id nil)) 161 | ([id opts*] 162 | (let [opts (queue-opts opts*) 163 | path (queue-path {:id id :opts opts}) 164 | q (queue/make path opts) 165 | i (last-index q)] 166 | {:type ::queue 167 | :id id 168 | :opts opts 169 | :controller (controllers/lookup path #(atom i)) 170 | :queue-impl q}))) 171 | 172 | (declare prime-tailer) 173 | 174 | (defn- tailer-id 175 | [queue id] 176 | (if id 177 | (combined-id id (:id queue)) 178 | (->> (cutil/uuid) 179 | (str "tailer") 180 | (keyword)))) 181 | 182 | (defn- tailer-opts 183 | [id tid] 184 | (when (and id tid) 185 | {:id (str "t" (hash tid))})) 186 | 187 | (defn tailer 188 | "Creates a tailer. 189 | 190 | Providing an id enables tailer position on the queue to persist 191 | across restarts. You can also optionally provide an unblock 192 | reference that when set to true will prevent the tailer from 193 | blocking. This is typically used to unblock and clean up blocked 194 | processor threads." 195 | ([queue] (tailer queue nil nil)) 196 | ([queue id] (tailer queue id nil)) 197 | ([queue id unblock] 198 | {:pre [(queue? queue)]} 199 | (let [tid (tailer-id queue id) 200 | opts (tailer-opts id tid) 201 | t (tail/make (:queue-impl queue) opts)] 202 | (prime-tailer 203 | {:type ::tailer 204 | :id tid 205 | :unblock unblock 206 | :queue queue 207 | :dirty (atom false) 208 | :persistent (boolean id) 209 | :tailer-impl t})))) 210 | 211 | (defn appender 212 | [{q :queue-impl :as queue}] 213 | {:pre [(queue? queue)]} 214 | {:type ::appender 215 | :queue queue 216 | :appender-impl (app/make q)}) 217 | 218 | (def queue-closed? 219 | (comp queue/closed? :queue-impl)) 220 | 221 | (defn set-direction 222 | "Sets the direction of the tailer to either :forward 223 | or :backward. Note: after changing the direction on the tailer you 224 | must do a read before you can measure the index again via 225 | `cues.queue/index`. This is an artifact of ChronicleQueue 226 | behaviour." 227 | [{t :tailer-impl 228 | d :dirty 229 | :as tailer} direction] 230 | {:pre [(tailer? tailer)]} 231 | (tail/set-direction! t direction) 232 | (reset! d true) 233 | tailer) 234 | 235 | (defn to-end 236 | "Moves the tailer to the end of the queue." 237 | [{t :tailer-impl 238 | d :dirty 239 | :as tailer}] 240 | {:pre [(tailer? tailer)]} 241 | (tail/to-end! t) 242 | (reset! d false) 243 | tailer) 244 | 245 | (defn to-start 246 | "Moves the tailer to the beginning of the queue." 247 | [{t :tailer-impl 248 | d :dirty 249 | :as tailer}] 250 | {:pre [(tailer? tailer)]} 251 | (tail/to-start! t) 252 | (reset! d false) 253 | tailer) 254 | 255 | (defn index* 256 | "Gets the index at the tailer's current position. ChronicleQueue 257 | tailers do not update their current index after changing direction 258 | until AFTER the next read. Cues guards against this edge case by 259 | throwing an error if you attempt to take the index before the next 260 | read." 261 | [{t :tailer-impl 262 | d :dirty 263 | :as tailer}] 264 | {:pre [(tailer? tailer)]} 265 | (if-not @d 266 | (tail/index t) 267 | (-> "Cannot take the index of a tailer after setting direction without first doing a read" 268 | (ex-info tailer) 269 | (throw)))) 270 | 271 | (def ^:dynamic index 272 | "Only rebind for testing!" 273 | index*) 274 | 275 | (def queue-obj 276 | "Returns the underlying ChronicleQueue object." 277 | (comp queue/underlying-queue :queue-impl)) 278 | 279 | (defn close-tailer! 280 | "Closes the given tailer." 281 | [tailer] 282 | {:pre [(tailer? tailer)]} 283 | (-> tailer 284 | (:tailer-impl) 285 | (tail/underlying-tailer) 286 | (.close)) 287 | tailer) 288 | 289 | (defn close-queue! 290 | "Closes the given queue." 291 | [{q :queue-impl 292 | :as queue}] 293 | {:pre [(queue? queue)]} 294 | (controllers/purge (queue-path queue)) 295 | (queue/close! q) 296 | (System/runFinalization)) 297 | 298 | (defn to-index* 299 | [{t :tailer-impl 300 | d :dirty 301 | :as tailer} i] 302 | {:pre [(tailer? tailer)]} 303 | (if (zero? i) 304 | (tail/to-start! t) 305 | (tail/to-index! t i)) 306 | (reset! d false) 307 | tailer) 308 | 309 | (def ^:dynamic to-index 310 | "Only rebind for testing!" 311 | to-index*) 312 | 313 | (defmacro with-tailer 314 | [bindings & body] 315 | (let [b (take-nth 2 bindings) 316 | q (take-nth 2 (rest bindings))] 317 | `(let ~(->> q 318 | (map #(list `tailer %)) 319 | (interleave b) 320 | (vec)) 321 | (try 322 | ~@body 323 | (finally 324 | (doseq [t# ~(vec b)] 325 | (close-tailer! t#))))))) 326 | 327 | (defn- prime-tailer 328 | "In certain rare cases, a newly initialized StoreTailer will start 329 | reading from the second index, but reports as if it had started at 330 | the first. Forcing the tailer explicitly to the index it reports to 331 | be at mitigates the issue. This is always done on tailer 332 | initialization." 333 | [tailer] 334 | {:pre [(tailer? tailer)]} 335 | (->> (index* tailer) 336 | (to-index* tailer))) 337 | 338 | (defn last-index* 339 | "A last index implementation that works for any kind of queue or 340 | appender. Note that appenders and queues will not necessarily return 341 | the same result, and appenders will throw an error if they have not 342 | appended anything yet." 343 | [x] 344 | (letfn [(last-index* [q] 345 | (.lastIndex 346 | ^SingleChronicleQueue 347 | (queue/underlying-queue q)))] 348 | (cond 349 | (satisfies? queue/IQueue x) (last-index* x) 350 | (satisfies? app/IAppender x) (app/last-index x) 351 | (appender? x) (app/last-index (:appender-impl x)) 352 | (queue? x) (last-index* (:queue-impl x))))) 353 | 354 | (def ^:dynamic last-index 355 | "Only rebind for testing!" 356 | last-index*) 357 | 358 | (defn last-read-index* 359 | [tailer] 360 | {:pre [(tailer? tailer)]} 361 | (let [^StoreTailer t (->> tailer 362 | (:tailer-impl) 363 | (tail/underlying-tailer))] 364 | (.lastReadIndex t))) 365 | 366 | (def ^:dynamic last-read-index 367 | "Only rebind for testing!" 368 | last-read-index*) 369 | 370 | (defn- dissoc-hash 371 | [msg] 372 | (if-let [m (-> msg 373 | (:q/meta) 374 | (dissoc :q/hash) 375 | (not-empty))] 376 | (assoc msg :q/meta m) 377 | (dissoc msg :q/meta))) 378 | 379 | (defn- materialize-meta 380 | [id tailer {m :q/meta :as msg}] 381 | (let [tx? (true? (get m :tx/t)) 382 | t? (true? (get-in m [:q/queue id :q/t])) 383 | t (when (or tx? t?) 384 | (last-read-index tailer))] 385 | (cond-> msg 386 | tx? (assoc-in [:q/meta :tx/t] t) 387 | t? (assoc-in [:q/meta :q/queue id :q/t] t)))) 388 | 389 | (defn- read-with-hash 390 | [{{id :id} :queue 391 | t-impl :tailer-impl 392 | dirty :dirty 393 | :as tailer}] 394 | {:pre [(tailer? tailer)]} 395 | (let [msg (tail/read! t-impl)] 396 | (reset! dirty false) 397 | (when msg 398 | (materialize-meta id tailer msg)))) 399 | 400 | (defn read 401 | "Reads message from tailer without blocking. Materializes metadata in 402 | message." 403 | [tailer] 404 | (-> tailer 405 | (read-with-hash) 406 | (dissoc-hash))) 407 | 408 | (defn peek 409 | "Like read, but does not advance tailer." 410 | [tailer] 411 | {:pre [(tailer? tailer)]} 412 | (when-let [msg (read tailer)] 413 | (->> tailer 414 | (last-read-index) 415 | (to-index tailer)) 416 | msg)) 417 | 418 | (defn mono-inc 419 | "Monotonically increases counter." 420 | [old new] 421 | (if (and (some? new) 422 | (or (nil? old) 423 | (< old new))) 424 | new 425 | old)) 426 | 427 | (defn- controller-inc! 428 | "Monotonically increases queue controller. It is okay to drop indexes, 429 | as long as each update is greater than or equal to the previous." 430 | [queue new-index] 431 | (update queue :controller swap! mono-inc new-index)) 432 | 433 | (def ^:dynamic written-index 434 | "Returns the index written by the appender. Only rebind for testing!" 435 | (fn [_ i] i)) 436 | 437 | (defn- lift-set 438 | [x] 439 | (if (set? x) x #{})) 440 | 441 | (defn- add-meta 442 | [{id :id 443 | {m :queue-meta} :opts} msg] 444 | (let [m* (lift-set m) 445 | t (get m* :q/t) 446 | tx (get m* :tx/t) 447 | ts (get m* :q/time)] 448 | (cond-> msg 449 | (false? m) (dissoc :q/meta) 450 | t (assoc-in [:q/meta :q/queue id :q/t] true) 451 | tx (assoc-in [:q/meta :tx/t] true) 452 | ts (assoc-in [:q/meta :q/queue id :q/time] 453 | (Instant/now))))) 454 | 455 | (defn- wrap-write 456 | [write-fn] 457 | (fn [{q :queue 458 | a :appender-impl 459 | :as appender} msg] 460 | {:pre [(appender? appender) 461 | (map? msg)]} 462 | (let [index (->> msg 463 | (add-meta q) 464 | (write-fn a))] 465 | (controller-inc! q index) 466 | (written-index q index)))) 467 | 468 | (defn write 469 | "The queue controller approximately follows the index of the queue: it 470 | can fall behind, but must be eventually consistent." 471 | [appender msg] 472 | (let [f (wrap-write app/write!)] 473 | (f appender msg))) 474 | 475 | (defn- watch-controller! 476 | [{id :id 477 | {c :controller} :queue 478 | :as tailer} continue] 479 | (when c 480 | (let [tailer-i (index tailer)] 481 | (add-watch c id 482 | (fn [id _ _ i] 483 | (when (<= tailer-i i) 484 | (deliver continue tailer))))))) 485 | 486 | (defn- watch-unblock! 487 | [{id :id 488 | unblock :unblock} continue] 489 | (when unblock 490 | (add-watch unblock id 491 | (fn [id _ _ new] 492 | (when (true? new) 493 | (deliver continue nil)))) 494 | (swap! unblock identity))) 495 | 496 | (defn- rm-watches 497 | [{id :id 498 | {c :controller} :queue 499 | unblock :unblock}] 500 | (when c 501 | (remove-watch c id)) 502 | (when unblock 503 | (remove-watch unblock id))) 504 | 505 | (defn continue? 506 | [{unblock :unblock 507 | :as obj}] 508 | (and (some? obj) 509 | (or (not (instance? IRef unblock)) 510 | (not @unblock)))) 511 | 512 | (defn- try-read 513 | [tailer] 514 | (when (continue? tailer) 515 | (loop [n 10000] 516 | (if (pos? n) 517 | (or (read tailer) 518 | (recur (dec n))) 519 | (throw (ex-info "Max read tries" {:tailer tailer})))))) 520 | 521 | (defn read!! 522 | "Blocking read. The tailer will consume eagerly if it can. If not, it 523 | will block until a new message is available. Implementation note: it 524 | is critical that the watch be placed before the first read. Note 525 | that the tailer index will always be ahead of the last index it 526 | read. Read will continue and return nil if at any point an unblock 527 | signal is received. This allows blocked threads to be cleaned up on 528 | unblock events." 529 | [tailer] 530 | {:pre [(tailer? tailer)]} 531 | (let [continue (promise)] 532 | (watch-controller! tailer continue) 533 | (watch-unblock! tailer continue) 534 | (if-let [msg (read tailer)] 535 | (do (rm-watches tailer) msg) 536 | (let [t @continue] 537 | (rm-watches t) 538 | (try-read t))))) 539 | 540 | (defn- alts 541 | [tailers] 542 | (some 543 | (fn [t] 544 | (when-let [msg (read t)] 545 | (doseq [t* tailers] 546 | (rm-watches t*)) 547 | [t msg])) 548 | tailers)) 549 | 550 | (defn alts!! 551 | "Completes at most one blocking read from several tailers. Semantics 552 | of each read are that or read!!." 553 | [tailers] 554 | {:pre [(every? tailer? tailers)]} 555 | (let [continue (promise)] 556 | (doseq [t tailers] 557 | (watch-controller! t continue) 558 | (watch-unblock! t continue)) 559 | (or (alts tailers) 560 | (let [t @continue] 561 | (doseq [t* tailers] 562 | (rm-watches t*)) 563 | (some->> (try-read t) 564 | (vector t)))))) 565 | 566 | (defmacro unhandled-error 567 | [msg & body] 568 | `(err/wrap-error 569 | {:q/type :q.type.err/unhandled-error 570 | :err/message ~msg} 571 | ~@body)) 572 | 573 | (defmulti processor 574 | "Analogous to Kafka processors. No default method." 575 | (constantly nil)) 576 | 577 | (defmulti persistent-snapshot 578 | "Persists processor tailer indices to a backing queue. Called once 579 | before each processor step. Taken together, persistent-snapshot, 580 | persistent-attempt, and persistent-recover implement message 581 | delivery semantics in Cues processors." 582 | :strategy) 583 | 584 | (defmulti persistent-snapshot-alts 585 | "Persists the alts processor tailer id to a backing queue. Called 586 | immediately after the processor alts read. Taken together, 587 | persistent-snapshot, persistent-attempt, and persistent-recover 588 | implement message delivery semantics in Cues processors." 589 | (fn [process _] 590 | (:strategy process))) 591 | 592 | (defmulti persistent-attempt 593 | "Persists message to processor output queues, as well as attempt data 594 | to backing queue. Called once during each processor step. Taken 595 | together, persistent-snapshot, persistent-attempt, and 596 | persistent-recover implement message delivery semantics in Cues 597 | processors." 598 | (fn [process _] 599 | (:strategy process))) 600 | 601 | (defmulti persistent-recover 602 | "Recovers processor tailer indices from backing queue once on 603 | processor start. Taken together, persistent-snapshot, 604 | persistent-attempt, and persistent-recover implement message 605 | delivery semantics in Cues processors." 606 | :strategy) 607 | 608 | (defn- snapshot? 609 | [msg] 610 | (= (:q/type msg) :q.type/snapshot)) 611 | 612 | (defn- snapshot-alts? 613 | [msg] 614 | (= (:q/type msg) :q.type/alts)) 615 | 616 | (defn- snapshot-map 617 | [id tailer-indices] 618 | {:q/type :q.type/snapshot 619 | :q/proc-id id 620 | :q/tailer-indices tailer-indices}) 621 | 622 | (defn- snapshot-alts-map 623 | [tailer-id] 624 | {:q/type :q.type/snapshot-alts 625 | :q/tailer-id tailer-id}) 626 | 627 | (defn- attempt-type 628 | [process] 629 | (if (= (:id (:errors process)) 630 | (:id (:queue (:appender process)))) 631 | :q.type/attempt-error 632 | :q.type/attempt-output)) 633 | 634 | (defn- attempt-map 635 | [process msg-index] 636 | {:q/type (attempt-type process) 637 | :q/message-index msg-index}) 638 | 639 | (defn- attempt-nil-map 640 | [attempt-hash] 641 | {:q/type :q.type/attempt-nil 642 | :q/hash attempt-hash}) 643 | 644 | (defn- error-config 645 | [{:keys [config strategy] :as process}] 646 | (-> config 647 | (select-keys [:id :topics :types :in :alts :out 648 | :errors :tailers :appenders]) 649 | (assoc :strategy strategy))) 650 | 651 | (defn- error-message 652 | [process msgs] 653 | {:q/type :q.type.err/processor 654 | :err.proc/config (error-config process) 655 | :err.proc/messages msgs}) 656 | 657 | (defmethod persistent-snapshot ::exactly-once 658 | [{uid :uid 659 | try-a :try-appender 660 | tailers :tailers 661 | retry :retry 662 | :as process}] 663 | {:pre [(appender? try-a)]} 664 | (let [m (->> tailers 665 | (map (juxt :id index)) 666 | (into {}) 667 | (snapshot-map uid))] 668 | (when-not retry (write try-a m)) 669 | (assoc process :delivery-hash (hash m)))) 670 | 671 | (defmethod persistent-snapshot-alts ::exactly-once 672 | [{try-a :try-appender 673 | retry :retry 674 | :as process} tailer] 675 | {:pre [(appender? try-a)]} 676 | (when-not retry 677 | (->> (:id tailer) 678 | (snapshot-alts-map) 679 | (write try-a))) 680 | process) 681 | 682 | (def ^:dynamic add-attempt-hash 683 | "Only rebind in testing." 684 | (fn [attempt-hash msg] 685 | (assoc-in msg [:q/meta :q/hash] attempt-hash))) 686 | 687 | (defn- encode-msg ^Bytes 688 | [appender msg attempt-hash] 689 | (let [codec (-> appender 690 | (:queue) 691 | (:queue-impl) 692 | (queue/codec))] 693 | (->> msg 694 | (add-attempt-hash attempt-hash) 695 | (codec/write codec) 696 | (Bytes/wrapForRead)))) 697 | 698 | (defn- appender-obj ^ExcerptAppender 699 | [appender] 700 | (-> appender 701 | (:appender-impl) 702 | (app/underlying-appender))) 703 | 704 | (def ^:dynamic attempt-index 705 | "Only rebind for tesitng!" 706 | (fn [_ i] i)) 707 | 708 | (defn- wrap-attempt 709 | [{a :appender 710 | try-a :try-appender 711 | h :delivery-hash 712 | :as process}] 713 | (fn [_ msg] 714 | (let [write-a (appender-obj a) 715 | msg* (encode-msg a msg h)] 716 | (with-open [doc (.writingDocument write-a)] 717 | (try 718 | (-> doc 719 | (.wire) 720 | (.write) 721 | (.bytes msg*)) 722 | (let [i (.index doc)] 723 | (->> i 724 | (attempt-index a) 725 | (attempt-map process) 726 | (write try-a)) 727 | i) 728 | (catch Throwable e 729 | (.rollbackOnClose doc) 730 | e)))))) 731 | 732 | (defn- wrap-throwable 733 | [write-fn] 734 | (fn [appender msg] 735 | (let [x (write-fn appender msg)] 736 | (when (instance? Throwable x) 737 | (throw x)) 738 | x))) 739 | 740 | (defn- attempt-full 741 | [{a :appender 742 | :as process} msg] 743 | (let [f (-> process 744 | (wrap-attempt) 745 | (wrap-throwable) 746 | (wrap-write))] 747 | (f a msg))) 748 | 749 | (defn- attempt-nil 750 | [{try-a :try-appender 751 | h :delivery-hash}] 752 | {:pre [(appender? try-a)]} 753 | (->> h 754 | (attempt-nil-map) 755 | (write try-a))) 756 | 757 | (defmethod persistent-attempt ::exactly-once 758 | [process msg] 759 | (err/wrap-error (error-message process msg) 760 | (if (and (:appender process) msg) 761 | (attempt-full process msg) 762 | (attempt-nil process)) 763 | process)) 764 | 765 | (defn- recover-tailers 766 | [{process-id :id 767 | tailers :tailers} snapshot] 768 | {:pre [(snapshot? snapshot)]} 769 | (doseq [{tid :id :as t} tailers] 770 | (if-let [i (-> snapshot 771 | (:q/tailer-indices) 772 | (get tid))] 773 | (to-index t i) 774 | (log/info "Topology changed, no snapshot for tailer" tid)))) 775 | 776 | (defn- recover-snapshot 777 | "Note that with respect to exactly-once delivery recover-tailers is 778 | idemopotent. We do not need to track if they have been reset we 779 | simply do it every time we detect the most recent delivery attempt 780 | has failed." 781 | [process {id :q/tailer-id} snapshot] 782 | (recover-tailers process snapshot) 783 | (assoc process 784 | :retry true 785 | :retry-tailer-id id)) 786 | 787 | (defn- next-snapshot 788 | "Reads back to the next snapshot on the try queue. Also return the 789 | last alts map if it is read along the way." 790 | [try-tailer] 791 | (loop [alts nil 792 | snapshot nil] 793 | (when-let [msg (read try-tailer)] 794 | (if (snapshot? msg) 795 | [alts msg] 796 | (if (snapshot-alts? msg) 797 | (recur msg snapshot) 798 | (recur alts snapshot)))))) 799 | 800 | (defn- recover-attempt-output 801 | [try-tailer {{q :queue} :appender 802 | :as process} {i :q/message-index}] 803 | (with-tailer [t q] 804 | (let [[alts snapshot] (next-snapshot try-tailer)] 805 | (when (not= (hash snapshot) 806 | (-> t 807 | (to-index i) 808 | (read-with-hash) 809 | (:q/meta) 810 | (:q/hash))) 811 | (recover-snapshot process alts snapshot))))) 812 | 813 | (defn- recover-attempt-error 814 | [try-tailer process msg] 815 | (let [p (->> (:errors process) 816 | (appender) 817 | (assoc process :appender))] 818 | (recover-attempt-output try-tailer p msg))) 819 | 820 | (defn- recover-attempt-nil 821 | [try-tailer process {h :q/hash}] 822 | (let [[alts snapshot] (next-snapshot try-tailer)] 823 | (when (not= h (hash snapshot)) 824 | (recover-snapshot process alts snapshot)))) 825 | 826 | (defn- recover-snapshot-alts 827 | [try-tailer process alts] 828 | (let [[_ snapshot] (next-snapshot try-tailer)] 829 | (recover-snapshot process alts snapshot))) 830 | 831 | (defn- recover 832 | [try-tailer process msg] 833 | (case (:q/type msg) 834 | :q.type/snapshot (recover-snapshot process nil msg) 835 | :q.type/snapshot-alts (recover-snapshot-alts try-tailer process msg) 836 | :q.type/attempt-error (recover-attempt-error try-tailer process msg) 837 | :q.type/attempt-output (recover-attempt-output try-tailer process msg) 838 | :q.type/attempt-nil (recover-attempt-nil try-tailer process msg))) 839 | 840 | (defmethod persistent-recover ::exactly-once 841 | [{{q :queue} :try-appender 842 | :as process}] 843 | (unhandled-error "tailer recovery" 844 | (with-tailer [t q] 845 | (set-direction t :backward) 846 | (or (some->> t 847 | (to-end) 848 | (read-with-hash) 849 | (recover t process)) 850 | process)))) 851 | 852 | (defmethod persistent-snapshot ::at-most-once 853 | [process] 854 | process) 855 | 856 | (defmethod persistent-snapshot-alts ::at-most-once 857 | [process _] 858 | process) 859 | 860 | (defmethod persistent-attempt ::at-most-once 861 | [{a :appender :as process} msg] 862 | (err/wrap-error (error-message process msg) 863 | (when (and a msg) 864 | (write a msg)) 865 | process)) 866 | 867 | (defmethod persistent-recover ::at-most-once 868 | [process] 869 | process) 870 | 871 | (defn- snapshot-unblock 872 | "Record tailer indices for unblock recovery." 873 | [{tailers :tailers 874 | :as process}] 875 | (->> tailers 876 | (map (juxt identity index)) 877 | (doall) 878 | (assoc process :snapshot-unblock))) 879 | 880 | (defn- recover-unblock 881 | "Reset tailer positions and interrupts the processor." 882 | ([process] 883 | (recover-unblock process (constantly true))) 884 | ([{snapshot :snapshot-unblock} pred?] 885 | (doseq [[tailer i] snapshot] 886 | (when (pred? tailer) 887 | (to-index tailer i))))) 888 | 889 | (defn- merge-deep 890 | [& args] 891 | (letfn [(f [& args] 892 | (if (every? map? args) 893 | (apply merge-with f args) 894 | (last args)))] 895 | (apply f (remove nil? args)))) 896 | 897 | (defn- merge-meta? 898 | [process] 899 | (get-in process [:queue-opts :queue-meta] true)) 900 | 901 | (defn- merge-meta-in 902 | [in] 903 | (->> (vals in) 904 | (keep :q/meta) 905 | (apply merge-deep))) 906 | 907 | (defn- assoc-meta-out 908 | [out m] 909 | (reduce-kv 910 | (fn [out k v] 911 | (cond-> out 912 | v (assoc k (update v :q/meta merge-deep m)))) 913 | {} 914 | out)) 915 | 916 | (defn- merge-meta 917 | [process in out] 918 | (when out 919 | (if (merge-meta? process) 920 | (if-let [m (merge-meta-in in)] 921 | (assoc-meta-out out m) 922 | out) 923 | out))) 924 | 925 | (defn- default-result-fn 926 | [process result] 927 | (get result (-> process 928 | (:appender) 929 | (:queue) 930 | (:id)))) 931 | 932 | (defn- default-run-fn 933 | [{f :fn 934 | appender :appender 935 | result-fn :result-fn 936 | :or {result-fn default-result-fn} 937 | :as process} in] 938 | (let [out (f process in)] 939 | (when appender 940 | (->> out 941 | (merge-meta process in) 942 | (result-fn process))))) 943 | 944 | (defn- processor-run 945 | [{:keys [run-fn error-fn] 946 | :or {run-fn default-run-fn} 947 | :as process} in] 948 | {:pre [error-fn run-fn]} 949 | (try 950 | (->> in 951 | (run-fn process) 952 | (persistent-attempt process)) 953 | (catch InterruptedException e (throw e)) 954 | (catch Exception e 955 | (error-fn process e) 956 | process))) 957 | 958 | (defn- zip-processor-read!! 959 | "Blocking read from all tailers into a map. Returns nil if any one of 960 | the blocking tailers returns nil." 961 | [{tailers :tailers}] 962 | (let [vals (map read!! tailers) 963 | keys (map (comp :id :queue) tailers)] 964 | (when (every? some? vals) 965 | (zipmap keys vals)))) 966 | 967 | (defn- alts-processor-tailers 968 | "On delivery retries, filters the list of tailers to the one that was 969 | previously read from." 970 | [{tailers :tailers 971 | retry-id :retry-tailer-id}] 972 | (cond->> tailers 973 | retry-id (filter (comp #{retry-id} :id)))) 974 | 975 | (defn- alts-processor-read!! 976 | [process] 977 | (let [tailers (alts-processor-tailers process)] 978 | (when-let [[t msg] (alts!! tailers)] 979 | (persistent-snapshot-alts process t) 980 | {(:id (:queue t)) msg}))) 981 | 982 | (defn- processor-read 983 | [{{alts :alts} :config 984 | read-fn :read-fn 985 | :as process}] 986 | (unhandled-error "processor read" 987 | (let [f (cond 988 | read-fn read-fn 989 | alts alts-processor-read!! 990 | :else zip-processor-read!!)] 991 | (f process)))) 992 | 993 | (defn- reset-processor 994 | [process] 995 | (dissoc process :retry :retry-tailer-id)) 996 | 997 | (defn- processor-step 998 | [process] 999 | (let [p (unhandled-error "processor snapshot" 1000 | (-> process 1001 | (snapshot-unblock) 1002 | (persistent-snapshot)))] 1003 | (if-let [in (processor-read p)] 1004 | (->> in 1005 | (processor-run p) 1006 | (reset-processor)) 1007 | (recover-unblock p)))) 1008 | 1009 | (defn log-processor-error 1010 | [{{:keys [id in alts out] 1011 | :or {in "source" 1012 | out "sink"}} :config} e] 1013 | (log/error e (format "%s (%s -> %s)" id (or alts in) out))) 1014 | 1015 | (defn- get-error-fn 1016 | [{q :errors}] 1017 | (if-let [a (some-> q appender)] 1018 | (fn [process e] 1019 | (let [p (assoc process :appender a)] 1020 | (log-processor-error process e) 1021 | (->> (err/error e) 1022 | (ex-data) 1023 | (persistent-attempt p)))) 1024 | (fn [process e] 1025 | (log-processor-error process e) 1026 | (throw e)))) 1027 | 1028 | (defn- wrap-processor-error 1029 | [handler] 1030 | (fn [process msgs] 1031 | (err/wrap-error (error-message process msgs) 1032 | (handler process msgs)))) 1033 | 1034 | (defn- rename-keys 1035 | [key-map m] 1036 | (cond-> m 1037 | key-map (set/rename-keys key-map))) 1038 | 1039 | (defn- wrap-msg-keys 1040 | [handler in-map out-map] 1041 | (fn [process msgs] 1042 | (some->> msgs 1043 | (rename-keys in-map) 1044 | (handler process) 1045 | (rename-keys out-map)))) 1046 | 1047 | (defn- wrap-msg-filter 1048 | [handler filter-fn] 1049 | (fn [process msgs] 1050 | (some->> msgs 1051 | (filter-fn) 1052 | (handler process)))) 1053 | 1054 | (defn- wrap-guard-no-out 1055 | [handler] 1056 | (fn [process msgs] 1057 | (let [r (handler process msgs)] 1058 | (when (-> process 1059 | (:config) 1060 | (:out)) 1061 | r)))) 1062 | 1063 | (defn- wrap-imperative 1064 | [handler in-map out-map] 1065 | (fn [{{a :appenders 1066 | t :tailers} :imperative 1067 | :as process} msgs] 1068 | (cond-> (dissoc process :tailers :appenders) 1069 | t (assoc :tailers (rename-keys in-map t)) 1070 | a (assoc :appenders (rename-keys out-map a)) 1071 | true (handler msgs)))) 1072 | 1073 | (defn- wrap-select-processor 1074 | [handler] 1075 | (fn [process msgs] 1076 | (-> process 1077 | (select-keys [:id 1078 | :config 1079 | :tailers 1080 | :appenders 1081 | :errors 1082 | :strategy 1083 | :delivery-hash 1084 | :system 1085 | :opts]) 1086 | (handler msgs)))) 1087 | 1088 | (defn- topics-filter 1089 | [values msgs] 1090 | (reduce-kv 1091 | (fn [m id {t :q/topics :as msg}] 1092 | (if (and (map? t) (some t values)) 1093 | (assoc m id msg) 1094 | m)) 1095 | nil 1096 | msgs)) 1097 | 1098 | (defn- types-filter 1099 | [values msgs] 1100 | (->> msgs 1101 | (filter #(some #{(:q/type (val %))} values)) 1102 | (into {}) 1103 | (not-empty))) 1104 | 1105 | (defn- msg-filter-fn 1106 | [{:keys [types topics]}] 1107 | (cond 1108 | topics (partial topics-filter topics) 1109 | types (partial types-filter types) 1110 | :else identity)) 1111 | 1112 | (defn- get-handler 1113 | [{f :fn 1114 | id :id}] 1115 | (or (get-method processor (or f id)) 1116 | (-> "Could not resolve processor :fn" 1117 | (ex-info {:id id}) 1118 | (throw)))) 1119 | 1120 | (defn- get-processor-fn 1121 | [{{:keys [in alts out tailers appenders] 1122 | :as config} :config}] 1123 | (let [rev (set/map-invert (or alts in)) 1124 | t (set/map-invert tailers) 1125 | a (set/map-invert appenders) 1126 | filter-fn (msg-filter-fn config)] 1127 | (-> config 1128 | (get-handler) 1129 | (wrap-select-processor) 1130 | (wrap-imperative t a) 1131 | (wrap-guard-no-out) 1132 | (wrap-msg-filter filter-fn) 1133 | (wrap-msg-keys rev out) 1134 | (wrap-processor-error)))) 1135 | 1136 | (defn- processor-handlers 1137 | [process] 1138 | (unhandled-error "building processor handlers" 1139 | (-> process 1140 | (assoc :error-fn (get-error-fn process)) 1141 | (assoc :fn (get-processor-fn process))))) 1142 | 1143 | (defn- close-tailers! 1144 | [process] 1145 | (doseq [t (:tailers process)] 1146 | (close-tailer! t)) 1147 | (doseq [t (-> process 1148 | (:imperative) 1149 | (:tailers) 1150 | (vals))] 1151 | (close-tailer! t))) 1152 | 1153 | (defn- processor-loop 1154 | [{:keys [id] :as process}] 1155 | (try 1156 | (loop [p (-> process 1157 | (processor-handlers) 1158 | (persistent-recover))] 1159 | (when (continue? p) 1160 | (recur (processor-step p)))) 1161 | (catch InterruptedException e (throw e)) 1162 | (catch Exception e 1163 | (log/error e "Processor: exit on unhandled error" id) 1164 | (throw e)) 1165 | (finally 1166 | (close-tailers! process)))) 1167 | 1168 | (defn- join-tailers 1169 | [{:keys [uid queue/in]} unblock] 1170 | (some->> in 1171 | (util/seqify) 1172 | (map #(tailer % uid unblock)) 1173 | (sort-by :id) 1174 | (doall))) 1175 | 1176 | (defn- fork-appenders 1177 | [{:keys [queue/out]}] 1178 | (some->> out 1179 | (util/seqify) 1180 | (map appender) 1181 | (doall))) 1182 | 1183 | (defn- backing-queue 1184 | [{:keys [uid] :as process}] 1185 | (let [p (-> process 1186 | (:queue-opts) 1187 | (:queue-path))] 1188 | (queue uid {:transient true 1189 | :queue-path p}))) 1190 | 1191 | (defn- start-impl 1192 | [process unblock start-fn] 1193 | (let [q (backing-queue process)] 1194 | {:try-queues [q] 1195 | :futures [(start-fn process unblock q)]})) 1196 | 1197 | (defn- start-join 1198 | "Reads messages from a seq of input tailers, applies a processor fn, 1199 | and writes the result to a single output appender. While queues can 1200 | be shared across threads, appenders and tailers cannot." 1201 | [{:keys [queue/out] 1202 | :as process} unblock try-queue] 1203 | (future 1204 | (processor-loop 1205 | (assoc process 1206 | :unblock unblock 1207 | :tailers (join-tailers process unblock) 1208 | :appender (appender out) 1209 | :try-appender (appender try-queue))))) 1210 | 1211 | (defn- fork-read!! 1212 | [{[tailer] :tailers}] 1213 | (read!! tailer)) 1214 | 1215 | (defn- fork-run-fn 1216 | "Conditionally passes input message from one input queue, usually a 1217 | backing queue, onto to the fork output queue, iff a message for the 1218 | output queue exists in the map." 1219 | [process in] 1220 | (->> process 1221 | (:queue/out) 1222 | (:id) 1223 | (get in))) 1224 | 1225 | (defn- start-jf-join 1226 | [process unblock fork-queue] 1227 | (-> process 1228 | (assoc :queue/out fork-queue 1229 | :result-fn (fn [_ r] r)) 1230 | (start-impl unblock start-join))) 1231 | 1232 | (defn- start-jf-fork 1233 | "Always remove :errors queues: all exceptions must be unhandled." 1234 | [{:keys [id uid queue/out] 1235 | :as process} unblock fork-queue] 1236 | (doall 1237 | (for [{oid :id :as o} out] 1238 | (-> (dissoc process :errors) 1239 | (assoc :id (combined-id id oid) 1240 | :uid (combined-id uid oid) 1241 | :run-fn fork-run-fn 1242 | :read-fn fork-read!! 1243 | :queue/in fork-queue 1244 | :queue/out o) 1245 | (start-impl unblock start-join))))) 1246 | 1247 | (defn- fork-id 1248 | [process] 1249 | (combined-id (:uid process) :fork)) 1250 | 1251 | (defn- start-join-fork 1252 | "Many to many processors are modelled as a single join loop connected 1253 | to one or more fork loops via a backing queue. Once a join loop has 1254 | committed to the delivery of the output messages, each fork 1255 | processing loop must deliver the message or raise an unhandled 1256 | exception. This ensures that while not all messages will arrive at 1257 | the same time, the delivery will be atomic." 1258 | [process unblock] 1259 | (let [q (->> (fork-id process) 1260 | (assoc process :uid) 1261 | (backing-queue)) 1262 | j (start-jf-join process unblock q) 1263 | f (start-jf-fork process unblock q)] 1264 | (-> (apply merge-with concat j f) 1265 | (assoc :fork-queue q)))) 1266 | 1267 | (defn- zip 1268 | [xs] 1269 | (-> (map (comp :id :queue) xs) 1270 | (zipmap xs) 1271 | (not-empty))) 1272 | 1273 | (defn- imp-primitives 1274 | [{uid :uid 1275 | t :queue/tailers 1276 | a :queue/appenders 1277 | :as process} unblock] 1278 | (let [p {:uid (suffix-id uid "-i") 1279 | :queue/in t 1280 | :queue/out a}] 1281 | {:tailers (zip (join-tailers p unblock)) 1282 | :appenders (zip (fork-appenders p))})) 1283 | 1284 | (defn- start-imperative 1285 | [{:keys [queue/out] 1286 | :as process} unblock try-queue] 1287 | (future 1288 | (processor-loop 1289 | (assoc process 1290 | :unblock unblock 1291 | :tailers (join-tailers process unblock) 1292 | :appender (when out (appender out)) 1293 | :try-appender (appender try-queue) 1294 | :imperative (imp-primitives process unblock))))) 1295 | 1296 | (defn- start-sink 1297 | [process unblock try-queue] 1298 | (future 1299 | (processor-loop 1300 | (assoc process 1301 | :unblock unblock 1302 | :tailers (join-tailers process unblock) 1303 | :try-appender (appender try-queue))))) 1304 | 1305 | (derive ::source ::processor) 1306 | (derive ::sink ::processor) 1307 | (derive ::join ::processor) 1308 | (derive ::join-fork ::processor) 1309 | (derive ::imperative ::processor) 1310 | 1311 | (defn processor? 1312 | [x] 1313 | (isa? (:type x) ::processor)) 1314 | 1315 | (defmulti start-processor-impl 1316 | :type) 1317 | 1318 | (defmethod start-processor-impl ::source 1319 | [{:keys [queue/out] :as process}] 1320 | {:appender (appender out)}) 1321 | 1322 | (defmethod start-processor-impl ::sink 1323 | [{:keys [id] :as process}] 1324 | (let [unblock (atom nil)] 1325 | (-> process 1326 | (start-impl unblock start-sink) 1327 | (assoc :unblock unblock)))) 1328 | 1329 | (defmethod start-processor-impl ::join 1330 | [{:keys [id] :as process}] 1331 | (let [unblock (atom nil)] 1332 | (-> process 1333 | (start-impl unblock start-join) 1334 | (assoc :unblock unblock)))) 1335 | 1336 | (defmethod start-processor-impl ::join-fork 1337 | [process] 1338 | (let [unblock (atom nil)] 1339 | (-> process 1340 | (start-join-fork unblock) 1341 | (assoc :unblock unblock)))) 1342 | 1343 | (defmethod start-processor-impl ::imperative 1344 | [{:keys [id] :as process}] 1345 | (let [unblock (atom nil)] 1346 | (-> process 1347 | (start-impl unblock start-imperative) 1348 | (assoc :unblock unblock)))) 1349 | 1350 | (defn- ensure-done 1351 | [{:keys [impl] :as process}] 1352 | (when (and impl @impl) 1353 | (doseq [r (:futures @impl)] 1354 | (try @r (catch Exception e)))) 1355 | process) 1356 | 1357 | (defn start-processor! 1358 | [{:keys [id state impl] 1359 | :as process}] 1360 | (if (compare-and-set! state nil ::started) 1361 | (do (log/info "Starting" id) 1362 | (->> (ensure-done process) 1363 | (start-processor-impl) 1364 | (reset! impl)) 1365 | process) 1366 | process)) 1367 | 1368 | (defn stop-processor! 1369 | [{:keys [id state impl] 1370 | :as process}] 1371 | (if (compare-and-set! state ::started nil) 1372 | (do (log/info "Stopping" id) 1373 | (some-> impl 1374 | (deref) 1375 | (:unblock) 1376 | (reset! true)) 1377 | (ensure-done process)) 1378 | process)) 1379 | 1380 | (defn- get-one-q 1381 | [g id] 1382 | (get-in g [:queues id])) 1383 | 1384 | (defn- get-q 1385 | [g ids] 1386 | (if (coll? ids) 1387 | (map (partial get-one-q g) ids) 1388 | (get-one-q g ids))) 1389 | 1390 | (defn- build-config 1391 | [{system :system 1392 | strategy :strategy 1393 | opts :queue-opts 1394 | :or {strategy ::exactly-once}} process] 1395 | (cond-> process 1396 | system (assoc :system system) 1397 | strategy (assoc :strategy strategy) 1398 | opts (assoc :queue-opts (::default opts)) 1399 | true (assoc :state (atom nil)) 1400 | true (assoc :impl (atom nil)))) 1401 | 1402 | (defn- build-processor 1403 | [{g-e :errors 1404 | :as g} {id :id 1405 | in :bind/in 1406 | out :bind/out 1407 | t :bind/tailers 1408 | a :bind/appenders 1409 | e :errors 1410 | :or {e g-e} 1411 | :as process}] 1412 | (cond-> process 1413 | id (assoc :uid (combined-id (:id g) id)) 1414 | e (assoc :errors (get-q g e)) 1415 | in (assoc :queue/in (get-q g in)) 1416 | out (assoc :queue/out (get-q g out)) 1417 | t (assoc :queue/tailers (get-q g t)) 1418 | a (assoc :queue/appenders (get-q g a)))) 1419 | 1420 | (defn- build-processors 1421 | [g processors] 1422 | (->> processors 1423 | (map (partial build-processor g)) 1424 | (map (partial build-config g)) 1425 | (map (juxt :id identity)) 1426 | (update g :processors cutil/into-once))) 1427 | 1428 | (defn- collect-processor-queues 1429 | [{:keys [errors] 1430 | :bind/keys [in out appenders tailers]}] 1431 | (concat (util/seqify in) 1432 | (util/seqify out) 1433 | (util/seqify appenders) 1434 | (util/seqify tailers) 1435 | (util/seqify errors))) 1436 | 1437 | (defn- build-queue 1438 | [{opts :queue-opts :as g} id] 1439 | (when (and id (not (get-one-q g id))) 1440 | (->> (get opts id) 1441 | (merge (::default opts)) 1442 | (queue id)))) 1443 | 1444 | (defn- build-queues 1445 | [{:keys [errors] :as g} processors] 1446 | (->> processors 1447 | (mapcat collect-processor-queues) 1448 | (cons errors) 1449 | (distinct) 1450 | (keep (partial build-queue g)) 1451 | (map (juxt :id identity)) 1452 | (update g :queues cutil/into-once))) 1453 | 1454 | (defn- queue-ids 1455 | [x] 1456 | (->> x 1457 | (util/seqify) 1458 | (map :id) 1459 | (map #(suffix-id % "-q")) 1460 | (set))) 1461 | 1462 | (defn- remove-queues 1463 | [p deps] 1464 | (->> (:nodes deps) 1465 | (map (fn [n] [(deps/two-hops deps n) n])) 1466 | (filter (fn [[_ n]] (contains? p n))) 1467 | (into {}) 1468 | (deps/graph))) 1469 | 1470 | (defn- processor-edges 1471 | [{id :id 1472 | in :queue/in 1473 | out :queue/out 1474 | a :queue/appenders 1475 | t :queue/tailers}] 1476 | (cond-> {} 1477 | in (assoc (queue-ids in) id) 1478 | out (assoc id (queue-ids out)) 1479 | t (assoc (queue-ids t) id) 1480 | a (assoc id (queue-ids a)))) 1481 | 1482 | (defn- build-topology* 1483 | [processors] 1484 | (->> processors 1485 | (vals) 1486 | (map processor-edges) 1487 | (set) 1488 | (deps/graph) 1489 | (remove-queues processors))) 1490 | 1491 | (defn- build-topology 1492 | [graph] 1493 | (->> (:processors graph) 1494 | (build-topology*) 1495 | (assoc graph :topology))) 1496 | 1497 | (defn- build-graph 1498 | [{p :processors :as g}] 1499 | (-> g 1500 | (dissoc :processors) 1501 | (build-queues p) 1502 | (build-processors p) 1503 | (build-topology))) 1504 | 1505 | (defn- parse-bindings 1506 | [bindings] 1507 | (let [b (vals bindings)] 1508 | (case (count b) 1509 | 1 (first b) 1510 | b))) 1511 | 1512 | (defn- parse-processor 1513 | [[t {:keys [id in out alts tailers appenders errors opts] 1514 | :as config}]] 1515 | (cutil/some-entries 1516 | (case t 1517 | ::source {:id id 1518 | :type t 1519 | :bind/out id 1520 | :opts opts} 1521 | {:id id 1522 | :type t 1523 | :bind/in (parse-bindings (or alts in)) 1524 | :bind/out (parse-bindings out) 1525 | :bind/tailers (parse-bindings tailers) 1526 | :bind/appenders (parse-bindings appenders) 1527 | :errors errors 1528 | :config config 1529 | :opts opts}))) 1530 | 1531 | (defn- coerce-many-cardinality 1532 | [[t form]] 1533 | (case t 1534 | :one [form] 1535 | :many form)) 1536 | 1537 | (defn- distinct-vals? 1538 | [m] 1539 | (apply distinct? (vals m))) 1540 | 1541 | (s/def ::fn qualified-keyword?) 1542 | (s/def ::id qualified-keyword?) 1543 | 1544 | (s/def ::id-one 1545 | (s/and (s/map-of keyword ::id :count 1) 1546 | distinct-vals?)) 1547 | 1548 | (s/def ::id-many 1549 | (s/and (s/map-of keyword ::id :min-count 2) 1550 | distinct-vals?)) 1551 | 1552 | (s/def :any/in 1553 | (s*/non-conformer 1554 | (s/or :one ::id-one 1555 | :many ::id-many))) 1556 | 1557 | (s/def :one/in ::id-one) 1558 | (s/def :one/out ::id-one) 1559 | (s/def :any/out :any/in) 1560 | (s/def :many/out ::id-many) 1561 | (s/def :many/alts ::id-many) 1562 | (s/def ::tailers :any/in) 1563 | (s/def ::appenders :any/out) 1564 | 1565 | (s/def ::types 1566 | (s*/conform-to 1567 | (s/or :one ::id 1568 | :many (s/coll-of ::id)) 1569 | coerce-many-cardinality)) 1570 | 1571 | (s/def ::topics ::types) 1572 | 1573 | (s/def ::generic 1574 | (s/keys :opt-un [::fn ::types ::topics])) 1575 | 1576 | (s/def ::sink 1577 | (s/merge (s/keys :req-un [(or :many/alts :any/in)]) 1578 | ::generic)) 1579 | 1580 | (s/def ::join 1581 | (s/merge (s/keys :req-un [(or :many/alts :any/in) :one/out]) 1582 | ::generic)) 1583 | 1584 | (s/def ::join-fork 1585 | (s/merge (s/keys :req-un [(or :many/alts :any/in) :many/out]) 1586 | ::generic)) 1587 | 1588 | (s/def ::imperative 1589 | (s/merge (s/keys :opt-un [:one/out] 1590 | :req-un [(or :many/alts :any/in) 1591 | (or ::tailers ::appenders)]) 1592 | ::generic)) 1593 | 1594 | (s/def ::source 1595 | (s/and (s/keys :req-un [::id]) 1596 | #(not (contains? % :in)) 1597 | #(not (contains? % :out)))) 1598 | 1599 | (s/def ::alts-or-in 1600 | #(not (and (contains? % :alts) 1601 | (contains? % :in)))) 1602 | 1603 | (s/def ::processor 1604 | (s/and ::alts-or-in 1605 | (s/or ::imperative ::imperative 1606 | ::join-fork ::join-fork 1607 | ::join ::join 1608 | ::source ::source 1609 | ::sink ::sink))) 1610 | 1611 | (s/def ::processors 1612 | (s/coll-of ::processor :kind sequential?)) 1613 | 1614 | (s/def ::graph 1615 | (s/keys :req-un [::processors ::id])) 1616 | 1617 | (defn graph 1618 | [g] 1619 | (-> (cutil/parse ::graph g) 1620 | (update :processors (partial map parse-processor)) 1621 | (assoc :config g) 1622 | (build-graph))) 1623 | 1624 | (defn topology 1625 | [graph] 1626 | (:topology graph)) 1627 | 1628 | (defn isomorphic? 1629 | [g1 g2] 1630 | (= (topology g1) 1631 | (topology g2))) 1632 | 1633 | (defn start-graph! 1634 | [graph] 1635 | (->> (partial cutil/map-vals start-processor!) 1636 | (update graph :processors))) 1637 | 1638 | (defn stop-graph! 1639 | [graph] 1640 | (->> (partial cutil/map-vals stop-processor!) 1641 | (update graph :processors))) 1642 | 1643 | (defn get-try-queues 1644 | [process] 1645 | (some-> process :impl deref :try-queues)) 1646 | 1647 | (defn get-fork-queue 1648 | [process] 1649 | (some-> process :impl deref :fork-queue)) 1650 | 1651 | (defn collect-graph-queues 1652 | [{:keys [processors queues]}] 1653 | (let [p (vals processors) 1654 | xs (concat (keep :errors p) 1655 | (keep get-fork-queue p) 1656 | (mapcat get-try-queues p) 1657 | (vals queues))] 1658 | (cutil/distinct-by :id xs))) 1659 | 1660 | (defn close-graph! 1661 | [graph] 1662 | (stop-graph! graph) 1663 | (->> graph 1664 | (collect-graph-queues) 1665 | (map close-queue!) 1666 | (dorun)) 1667 | graph) 1668 | 1669 | (defn send! 1670 | ([graph msg] 1671 | (send! graph nil msg)) 1672 | ([{p :processors 1673 | config :config} source msg] 1674 | (let [id (or source (:source config)) 1675 | {impl :impl 1676 | type :type} (get p id)] 1677 | (if (= type ::source) 1678 | (write (:appender @impl) msg) 1679 | (throw (ex-info "Could not find source" {:id id})))))) 1680 | 1681 | (defn messages 1682 | "Returns a lazy list of all remaining messages." 1683 | [tailer] 1684 | (->> (partial read tailer) 1685 | (repeatedly) 1686 | (take-while some?))) 1687 | 1688 | (defn all-messages 1689 | "Eagerly gets all messages in the cue. Could be many!" 1690 | [queue] 1691 | (with-tailer [t queue] 1692 | (doall (messages t)))) 1693 | 1694 | (defn graph-messages 1695 | "Eagerly gets all messages in the queue. Could be many!" 1696 | [{queues :queues} id] 1697 | (-> queues 1698 | (get id) 1699 | (all-messages))) 1700 | 1701 | (defn all-graph-messages 1702 | "Eagerly gets all messages in the graph. Could be many!" 1703 | [{queues :queues}] 1704 | (let [queues* (vals queues)] 1705 | (zipmap 1706 | (map :id queues*) 1707 | (map all-messages queues*)))) 1708 | 1709 | ;; Data file management 1710 | 1711 | (defn delete-queue! 1712 | "Deletes the queue data on disk, prompting by default. 1713 | 1714 | Implementation note: must also purge the queue controller or 1715 | blocking will break the next time the queue is made." 1716 | ([queue] 1717 | (delete-queue! queue false)) 1718 | ([queue force] 1719 | {:pre [(queue? queue)]} 1720 | (let [p (queue-path queue)] 1721 | (when (or force (cutil/prompt-delete! p)) 1722 | (close-queue! queue) 1723 | (controllers/purge p) 1724 | (-> p 1725 | (io/file) 1726 | (cutil/delete-file)))))) 1727 | 1728 | (defn close-and-delete-try-queues! 1729 | [g] 1730 | (doseq [q (->> (:processors g) 1731 | (vals) 1732 | (mapcat get-try-queues))] 1733 | (stop-graph! g) 1734 | (close-graph! g) 1735 | (delete-queue! q true))) 1736 | 1737 | (defn close-and-delete-graph! 1738 | "Closes the graph and all queues and deletes all queue data." 1739 | ([g] 1740 | (close-and-delete-graph! g false)) 1741 | ([g force] 1742 | (let [queues (collect-graph-queues g)] 1743 | (when (or force (-> (count queues) 1744 | (str " queues") 1745 | (cutil/prompt-delete!))) 1746 | (stop-graph! g) 1747 | (close-graph! g) 1748 | (doseq [q queues] 1749 | (delete-queue! q true)))))) 1750 | 1751 | (defn delete-all-queues! 1752 | "Deletes all queue data at either the provided or default path. 1753 | 1754 | Implementation note: must also purge the queue controller or 1755 | blocking will break the next time the queue is made." 1756 | ([] 1757 | (delete-all-queues! queue-path-default)) 1758 | ([queue-path] 1759 | (when (cutil/prompt-delete! queue-path) 1760 | (controllers/purge queue-path) 1761 | (-> queue-path 1762 | (io/file) 1763 | (cutil/delete-file))))) 1764 | 1765 | (defn unused-queue-files 1766 | [queue] 1767 | (let [p (queue-path queue)] 1768 | (-> (io/file p) 1769 | (FileUtil/removableRollFileCandidates) 1770 | (.collect (Collectors/toList))))) 1771 | 1772 | (defn delete-unused-queue-files! 1773 | [queue] 1774 | (doseq [^File f (unused-queue-files queue)] 1775 | (.delete f))) 1776 | -------------------------------------------------------------------------------- /src/cues/test.clj: -------------------------------------------------------------------------------- 1 | (ns cues.test 2 | "Provides Cues test fixtures." 3 | (:require [cinch.core :as util] 4 | [cues.queue :as q] 5 | [taoensso.timbre :as log])) 6 | 7 | (defn with-warn 8 | [f] 9 | (log/with-level :warn 10 | (f))) 11 | 12 | (defn index-from-1 13 | "For testing purposes only." 14 | [tailer] 15 | (q/with-tailer [t-1 (:queue tailer)] 16 | (let [i (q/index* tailer) 17 | _ (q/to-start t-1) 18 | i-1 (q/index* t-1)] 19 | (if (zero? i) 1 (inc (- i i-1)))))) 20 | 21 | (defn last-index-from-1 22 | "For testing purposes only." 23 | [queue] 24 | (q/with-tailer [t-1 queue] 25 | (let [i (q/last-index* queue) 26 | _ (q/to-start t-1) 27 | i-1 (q/index* t-1)] 28 | (if (zero? i) 1 (inc (- i i-1)))))) 29 | 30 | (defn last-read-index-from-1 31 | "For testing purposes only." 32 | [tailer] 33 | (q/with-tailer [t-1 (:queue tailer)] 34 | (let [i (q/last-read-index* tailer) 35 | _ (q/to-start t-1) 36 | i-1 (q/index* t-1)] 37 | (if (zero? i) 1 (inc (- i i-1)))))) 38 | 39 | (defn to-index-from-1 40 | "For testing purposes only." 41 | [tailer i] 42 | (if (pos? i) 43 | (q/with-tailer [t-1 (:queue tailer)] 44 | (let [_ (q/to-start t-1) 45 | i-1 (q/index* t-1)] 46 | (->> (+ i (dec i-1)) 47 | (q/to-index* tailer)))) 48 | (q/to-start tailer))) 49 | 50 | (defn written-index-from-1 51 | "For testing purposes only." 52 | [queue i] 53 | (q/with-tailer [t-1 queue] 54 | (let [_ (q/to-start t-1) 55 | i-1 (q/index* t-1)] 56 | (inc (- i i-1))))) 57 | 58 | (defn attempt-index-from-1 59 | "For testing purposes only." 60 | [appender i] 61 | (q/with-tailer [t-1 (:queue appender)] 62 | (let [_ (q/to-start t-1) 63 | i-1 (q/index* t-1)] 64 | (if (zero? i) 1 (inc (- i i-1)))))) 65 | 66 | (defn with-test-indices 67 | "A number of cues.queue functions are rebound to make queue indices 68 | easier to work with in unit tests. While ChronicleQueue indicies are 69 | deterministic, they have a complex relationship to the roll cycles 70 | of the data on disk. Queue indices are guaranteed to increase 71 | monotonically, but not always continguously, and in general code 72 | that tries to predict future indices should be avoided. However, 73 | under the narrow constraints of the unit tests and test queue 74 | configurations, these new bindings will start all indices at 1 and 75 | then increase continguously. Just beware that this does NOT hold in 76 | general, and you should never rebind these methods outside of unit 77 | tests." 78 | [f] 79 | (binding [q/last-read-index last-read-index-from-1 80 | q/last-index last-index-from-1 81 | q/to-index to-index-from-1 82 | q/index index-from-1 83 | q/written-index written-index-from-1 84 | q/attempt-index attempt-index-from-1] 85 | (f))) 86 | 87 | (defmacro with-graph-and-delete 88 | [[sym :as binding] & body] 89 | `(let ~binding 90 | (let [~sym (-> ~sym 91 | (util/assoc-nil :errors ::error) 92 | (update :queue-opts util/assoc-nil ::q/default {:queue-meta #{:q/t}}) 93 | (q/graph) 94 | (q/start-graph!))] 95 | (try 96 | ~@body 97 | (finally 98 | (q/close-and-delete-graph! ~sym true)))))) 99 | 100 | (defn simplify-exceptions 101 | [messages] 102 | (map #(update % 103 | :err/cause 104 | select-keys 105 | [:cause]) 106 | messages)) 107 | -------------------------------------------------------------------------------- /src/cues/util.clj: -------------------------------------------------------------------------------- 1 | (ns cues.util 2 | (:require [clojure.java.io :as io] 3 | [clojure.spec.alpha :as s] 4 | [clojure.walk :as walk] 5 | [expound.alpha :as expound]) 6 | (:import java.io.File 7 | java.util.UUID)) 8 | 9 | (defn some-entries 10 | "Returns the given map with nil entries removed." 11 | [m] 12 | (reduce-kv 13 | (fn [m k v] 14 | (cond-> m 15 | (some? v) (assoc k v))) 16 | {} 17 | m)) 18 | 19 | (defn into-once 20 | "Like into, but throws an error on duplicate keys." 21 | [m coll] 22 | (reduce 23 | (fn [m [k v]] 24 | (if (contains? m k) 25 | (throw (ex-info "Already added" {:key k})) 26 | (assoc m k v))) 27 | (or m {}) 28 | (seq coll))) 29 | 30 | (defn prompt-delete! 31 | [p] 32 | (println "Delete data (yes/no)?" p) 33 | (case (read-line) 34 | "yes" true 35 | "no" false 36 | (println "Must be yes/no"))) 37 | 38 | (defn assert-path 39 | "Ensure path is relative and in project." 40 | [^File f] 41 | (let [p (.getCanonicalPath f) 42 | project-p (System/getProperty "user.dir") 43 | project-re (re-pattern (str "^" project-p))] 44 | (io/as-relative-path f) 45 | (when-not (re-find project-re p) 46 | (throw (ex-info "Path not in project" {:path p}))))) 47 | 48 | (defn delete-file 49 | "Deletes files in project only, recursively if directory." 50 | [^File f] 51 | (assert-path f) 52 | (when (.isDirectory f) 53 | (doseq [file (.listFiles f)] 54 | (delete-file file))) 55 | (when (.exists f) 56 | (io/delete-file f))) 57 | 58 | (defn list-files 59 | "Lists files in the directory, not recursive." 60 | [^File f] 61 | (->> f 62 | (.listFiles) 63 | (filter #(.isFile %)))) 64 | 65 | (defn uuid 66 | [] 67 | (UUID/randomUUID)) 68 | 69 | (defn map-vals 70 | [f m] 71 | (reduce-kv 72 | (fn [m k v] 73 | (assoc m k (f v))) 74 | {} 75 | m)) 76 | 77 | (defn distinct-by 78 | [f coll] 79 | (->> coll 80 | (group-by f) 81 | (map (comp first second)) 82 | (doall))) 83 | 84 | (defn dissoc-in 85 | [m [k & more]] 86 | (or (some->> more 87 | (dissoc-in (get m k)) 88 | (not-empty) 89 | (assoc m k)) 90 | (dissoc m k))) 91 | 92 | (defn parse 93 | [spec x] 94 | (let [form (s/conform spec x)] 95 | (if (= form ::s/invalid) 96 | (-> (expound/expound-str spec x) 97 | (IllegalArgumentException.) 98 | (throw)) 99 | form))) 100 | 101 | ;; Catalogs 102 | 103 | (defn bind-catalog 104 | [catalog bindings] 105 | (walk/postwalk 106 | (fn [x] 107 | (get bindings x x)) 108 | catalog)) 109 | 110 | (defn merge-catalogs 111 | [& catalogs] 112 | (->> catalogs 113 | (reverse) 114 | (apply concat) 115 | (distinct-by :id))) 116 | -------------------------------------------------------------------------------- /test/cues/deps_test.clj: -------------------------------------------------------------------------------- 1 | (ns cues.deps-test 2 | (:require [cues.deps :as deps] 3 | [clojure.test :as t :refer [is]] 4 | [com.stuartsierra.dependency :as sdeps])) 5 | 6 | (defn g 7 | [nodes dependencies dependents] 8 | (->> dependents 9 | (sdeps/->MapDependencyGraph dependencies) 10 | (deps/disconnected-graph nodes))) 11 | 12 | (t/deftest graph-test 13 | (is (= (deps/graph []) 14 | (deps/graph #{}) 15 | (deps/graph {}) 16 | (g #{} {} {}))) 17 | (is (= (deps/graph {[] []}) 18 | (g #{} {} {}))) 19 | (is (= (deps/graph {[nil] []}) 20 | (g #{} {} {}))) 21 | (is (= (deps/graph #{nil [nil]}) 22 | (g #{} {} {}))) 23 | (is (= (deps/graph #{::A ::B}) 24 | (deps/graph #{::A ::B nil}) 25 | (deps/graph #{[::A nil] ::B nil}) 26 | (deps/graph #{::A #{::B}}) 27 | (g #{::A ::B} {} {}))) 28 | (is (= (deps/graph #{::A {::B ::C}}) 29 | (g 30 | #{::A ::B ::C} 31 | {::C #{::B}} 32 | {::B #{::C}}))) 33 | (is (= (deps/graph [::A ::B ::C ::D]) 34 | (deps/graph [::A [::B [::C [::D]]]]) 35 | (deps/graph [::A {::B [::C #{::D}]}]) 36 | (g 37 | #{::A ::B ::C ::D} 38 | {::B #{::A} ::C #{::B} ::D #{::C}} 39 | {::A #{::B} ::B #{::C} ::C #{::D}}))) 40 | (is (= (deps/graph {::A ::B}) 41 | (deps/graph [::A ::B]) 42 | (g #{::A ::B} {::B #{::A}} {::A #{::B}}))) 43 | (is (= (deps/graph {::A [::B ::C]}) 44 | (g 45 | #{::A ::B ::C} 46 | {::B #{::A} ::C #{::B}} 47 | {::A #{::B} ::B #{::C}}))) 48 | (is (= (deps/graph {#{::A ::D} [::B ::C] 49 | ::B [::E]}) 50 | (g 51 | #{::A ::B ::C ::D ::E} 52 | {::B #{::D ::A} 53 | ::E #{::B} 54 | ::C #{::B}} 55 | {::A #{::B} 56 | ::D #{::B} 57 | ::B #{::C ::E}}))) 58 | (is (= (deps/graph [#{::A ::B} 59 | #{[::C ::D ::E] 60 | [::F ::G ::H]}]) 61 | (g 62 | #{::B ::C ::D ::G ::E ::F ::A ::H} 63 | {::D #{::C} 64 | ::E #{::D} 65 | ::G #{::F} 66 | ::H #{::G} 67 | ::C #{::B ::A} 68 | ::F #{::B ::A}} 69 | {::C #{::D} 70 | ::D #{::E} 71 | ::F #{::G} 72 | ::G #{::H} 73 | ::B #{::C ::F} 74 | ::A #{::C ::F}}))) 75 | (let [g (deps/graph [#{::A ::B} 76 | #{[::C ::D ::E] 77 | [::F ::G ::H]}]) 78 | compare (deps/topo-comparator g)] 79 | ;; The clojure.tools.namespace.dependency/topo-comparator 80 | ;; implementation chooses a 2-way comparator over a 3-way one, 81 | ;; maybe for performance reasons. This results in a correct sort, 82 | ;; but a non-deterministic order for certain types of 83 | ;; graphs. Therefore, here instead of checking the order of the 84 | ;; sort, these tests assert comparisons where the result is known 85 | ;; to be deterministic. 86 | (is (= [::C ::D ::E] (sort compare [::E ::D ::C]))) 87 | (is (= [::F ::G ::H] (sort compare [::H ::G ::F]))) 88 | (is (every? #{-1} (for [x [::A ::B] 89 | y [::C ::D ::E]] 90 | (compare x y)))) 91 | (is (every? #{1} (for [x [::C ::D ::E] 92 | y [::A ::B]] 93 | (compare x y)))) 94 | (is (every? #{-1} (for [x [::A ::B] 95 | y [::F ::G ::H]] 96 | (compare x y)))) 97 | (is (every? #{1} (for [x [::F ::G ::H] 98 | y [::A ::B]] 99 | (compare x y)))))) 100 | 101 | (t/deftest in-and-out-node-test 102 | (let [g (deps/graph #{})] 103 | (is (= (deps/out-nodes g) 104 | #{})) 105 | (is (= (deps/in-nodes g) 106 | #{}))) 107 | (let [g (deps/graph {::A #{::B ::C} 108 | #{::D ::C} #{::E}})] 109 | (is (= (deps/out-nodes g) 110 | #{::B ::E})) 111 | (is (= (deps/in-nodes g) 112 | #{::A ::D}))) 113 | (let [g (deps/graph [::A #{::B ::C} #{::D ::E}])] 114 | (is (= (deps/out-nodes g) 115 | #{::D ::E})) 116 | (is (= (deps/in-nodes g) 117 | #{::A})))) 118 | -------------------------------------------------------------------------------- /test/cues/queue_test.clj: -------------------------------------------------------------------------------- 1 | (ns cues.queue-test 2 | (:require [clojure.java.io :as io] 3 | [clojure.spec.alpha :as s] 4 | [clojure.test :as t :refer [is]] 5 | [cues.error :as err] 6 | [cues.queue :as q] 7 | [cues.test :as qt] 8 | [cues.util :as cutil] 9 | [taoensso.timbre :as log] 10 | 11 | ;; Configure logging for tests. 12 | [cues.log])) 13 | 14 | (t/use-fixtures :each 15 | (t/join-fixtures 16 | [qt/with-test-indices qt/with-warn])) 17 | 18 | (defn done? 19 | ([done] 20 | (done? done 1000)) 21 | ([done ms] 22 | (deref done ms false))) 23 | 24 | (defn throw-interrupt! 25 | [] 26 | (throw (InterruptedException. "Interrupted processor"))) 27 | 28 | (t/deftest parse-processor-impl-test 29 | (let [parse #(first (s/conform ::q/processor %))] 30 | (is (= (parse {:id ::source}) 31 | ::q/source)) 32 | (is (= (parse {:id ::processor 33 | :in {:in ::in}}) 34 | ::q/sink)) 35 | (is (= (parse {:id ::processor 36 | :in {:in ::in} 37 | :out {:out ::out}}) 38 | ::q/join)) 39 | (is (= (parse {:id ::processor 40 | :in {:in-1 ::in-1 41 | :in-2 ::in-2} 42 | :out {:out-1 ::out-1}}) 43 | ::q/join)) 44 | (is (= (parse {:id ::processor 45 | :in {:in ::in} 46 | :out {:out-1 ::out-1 47 | :out-2 ::out-2}}) 48 | ::q/join-fork)) 49 | (is (= (parse {:id ::processor 50 | :in {:in-1 ::in-1 51 | :in-2 ::in-2} 52 | :out {:out-1 ::out-1 53 | :out-2 ::out-2}}) 54 | ::q/join-fork)) 55 | (is (= (parse {:id ::processor 56 | :in {:in ::in} 57 | :out {:out-1 ::out-1} 58 | :appenders {:a ::appenders}}) 59 | ::q/imperative)) 60 | (is (= (parse {:id ::processor 61 | :in {:in ::in} 62 | :out {:out-1 ::out-1} 63 | :tailers {:t ::tailer}}) 64 | ::q/imperative)) 65 | (is (= (parse {:id ::processor 66 | :in {:in ::in} 67 | :out {:out-1 ::out-1} 68 | :tailers {:t ::tailer} 69 | :appenders {:a ::appenders}}) 70 | ::q/imperative)))) 71 | 72 | (defmethod q/processor ::map-reduce 73 | [{{r-fn :reduce-fn 74 | m-fn :map-fn 75 | bindings :to 76 | :or {m-fn identity}} :opts} msgs] 77 | (let [result {:x (->> (vals msgs) 78 | (map (comp m-fn :x)) 79 | (reduce r-fn))}] 80 | (zipmap bindings (repeat result)))) 81 | 82 | (defmethod q/processor ::done 83 | [{{:keys [done x-n]} :opts 84 | system :system} {{x :x} :in}] 85 | (when (and (= x x-n) done) 86 | (deliver done (or system true)))) 87 | 88 | (defmethod q/processor ::done-counter 89 | [{{:keys [done counter n]} :opts} _] 90 | (when (= n (swap! counter inc)) 91 | (deliver done true))) 92 | 93 | (t/deftest graph-system-test 94 | (let [done (promise)] 95 | (qt/with-graph-and-delete 96 | [g {:id ::graph 97 | :processors [{:id ::s1} 98 | {:id ::done 99 | :in {:in ::s1} 100 | :opts {:done done 101 | :x-n 1}}] 102 | :system {:component :running}}] 103 | (q/send! g ::s1 {:x 1}) 104 | (is (done? done)) 105 | (is (= @done {:component :running}))))) 106 | 107 | (t/deftest graph-source-sink-test 108 | (let [done (promise)] 109 | (qt/with-graph-and-delete 110 | [g {:id ::graph 111 | :processors [{:id ::s1} 112 | {:id ::done 113 | :in {:in ::s1} 114 | :opts {:done done 115 | :x-n 3}}]}] 116 | (q/send! g ::s1 {:x 1}) 117 | (q/send! g ::s1 {:x 2}) 118 | (q/send! g ::s1 {:x 3}) 119 | (is (done? done)) 120 | (is (= (q/all-graph-messages g) 121 | {::qt/error [] 122 | ::s1 [{:x 1 :q/meta {:q/queue {::s1 {:q/t 1}}}} 123 | {:x 2 :q/meta {:q/queue {::s1 {:q/t 2}}}} 124 | {:x 3 :q/meta {:q/queue {::s1 {:q/t 3}}}}]}))))) 125 | 126 | (t/deftest graph-pipe-test 127 | (let [done (promise)] 128 | (qt/with-graph-and-delete 129 | [g {:id ::graph 130 | :processors [{:id ::s1} 131 | {:id ::map-reduce 132 | :in {:in ::s1} 133 | :out {:out ::q1} 134 | :opts {:map-fn inc 135 | :to [:out]}} 136 | {:id ::done 137 | :in {:in ::q1} 138 | :opts {:done done 139 | :x-n 4}}] 140 | :queue-opts {::s1 {:queue-meta #{:q/t :tx/t}}}}] 141 | (q/send! g ::s1 {:x 1}) 142 | (q/send! g ::s1 {:x 2}) 143 | (q/send! g ::s1 {:x 3}) 144 | (is (done? done)) 145 | (is (= (q/all-graph-messages g) 146 | {::qt/error [] 147 | ::s1 [{:x 1 :q/meta {:tx/t 1 :q/queue {::s1 {:q/t 1}}}} 148 | {:x 2 :q/meta {:tx/t 2 :q/queue {::s1 {:q/t 2}}}} 149 | {:x 3 :q/meta {:tx/t 3 :q/queue {::s1 {:q/t 3}}}}] 150 | ::q1 [{:x 2 :q/meta {:tx/t 1 :q/queue {::s1 {:q/t 1} 151 | ::q1 {:q/t 1}}}} 152 | {:x 3 :q/meta {:tx/t 2 :q/queue {::s1 {:q/t 2} 153 | ::q1 {:q/t 2}}}} 154 | {:x 4 :q/meta {:tx/t 3 :q/queue {::s1 {:q/t 3} 155 | ::q1 {:q/t 3}}}}]}))))) 156 | 157 | (t/deftest graph-alts-test 158 | (let [d1 (promise) 159 | d2 (promise) 160 | d3 (promise) 161 | done (promise)] 162 | (qt/with-graph-and-delete 163 | [g {:id ::graph 164 | :processors [{:id ::s1} 165 | {:id ::s2} 166 | {:id ::alts 167 | :fn ::map-reduce 168 | :alts {:q1 ::s1 169 | :q2 ::s2} 170 | :out {:out ::q1} 171 | :opts {:map-fn (fn [x] 172 | (case x 173 | 1 (deliver d1 true) 174 | 2 (deliver d2 true) 175 | 3 (deliver d3 true) 176 | true) 177 | (inc x)) 178 | :to [:out]}} 179 | {:id ::done-counter 180 | :in {:in ::q1} 181 | :opts {:done done 182 | :counter (atom 0) 183 | :n 4}}]}] 184 | (q/send! g ::s1 {:x 1}) 185 | (is (done? d1)) 186 | (q/send! g ::s2 {:x 2}) 187 | (is (done? d2)) 188 | (q/send! g ::s2 {:x 3}) 189 | (is (done? d3)) 190 | (q/send! g ::s1 {:x 4}) 191 | (is (done? done)) 192 | (is (= (q/all-graph-messages g) 193 | {::qt/error [] 194 | ::s1 [{:x 1 :q/meta {:q/queue {::s1 {:q/t 1}}}} 195 | {:x 4 :q/meta {:q/queue {::s1 {:q/t 2}}}}] 196 | ::s2 [{:x 2 :q/meta {:q/queue {::s2 {:q/t 1}}}} 197 | {:x 3 :q/meta {:q/queue {::s2 {:q/t 2}}}}] 198 | ::q1 [{:x 2 :q/meta {:q/queue {::s1 {:q/t 1} 199 | ::q1 {:q/t 1}}}} 200 | {:x 3 :q/meta {:q/queue {::s2 {:q/t 1} 201 | ::q1 {:q/t 2}}}} 202 | {:x 4 :q/meta {:q/queue {::s2 {:q/t 2} 203 | ::q1 {:q/t 3}}}} 204 | {:x 5 :q/meta {:q/queue {::s1 {:q/t 2} 205 | ::q1 {:q/t 4}}}}]}))))) 206 | 207 | (defn- preserve-meta 208 | [in out] 209 | (assoc out :q/meta (:q/meta in))) 210 | 211 | (defmethod q/processor ::imperative 212 | [{{:keys [a]} :appenders} {msg :in}] 213 | (->> {:x (inc (:x msg))} 214 | (preserve-meta msg) 215 | (q/write a)) 216 | nil) 217 | 218 | (t/deftest graph-imperative-test 219 | (let [done (promise)] 220 | (qt/with-graph-and-delete 221 | [g {:id ::graph 222 | :processors [{:id ::s1} 223 | {:id ::imperative 224 | :in {:in ::s1} 225 | :appenders {:a ::q1}} 226 | {:id ::done 227 | :in {:in ::q1} 228 | :opts {:done done 229 | :x-n 4}}] 230 | :queue-opts {::s1 {:queue-meta #{:q/t :tx/t}}}}] 231 | (q/send! g ::s1 {:x 1}) 232 | (q/send! g ::s1 {:x 2}) 233 | (q/send! g ::s1 {:x 3}) 234 | (is (done? done)) 235 | (is (= (q/all-graph-messages g) 236 | {::qt/error [] 237 | ::s1 [{:x 1 :q/meta {:tx/t 1 :q/queue {::s1 {:q/t 1}}}} 238 | {:x 2 :q/meta {:tx/t 2 :q/queue {::s1 {:q/t 2}}}} 239 | {:x 3 :q/meta {:tx/t 3 :q/queue {::s1 {:q/t 3}}}}] 240 | ::q1 [{:x 2 :q/meta {:tx/t 1 :q/queue {::s1 {:q/t 1} 241 | ::q1 {:q/t 1}}}} 242 | {:x 3 :q/meta {:tx/t 2 :q/queue {::s1 {:q/t 2} 243 | ::q1 {:q/t 2}}}} 244 | {:x 4 :q/meta {:tx/t 3 :q/queue {::s1 {:q/t 3} 245 | ::q1 {:q/t 3}}}}]}))))) 246 | 247 | (t/deftest graph-join-test 248 | (let [done (promise)] 249 | (qt/with-graph-and-delete 250 | [g {:id ::graph 251 | :processors [{:id ::s1} 252 | {:id ::s2} 253 | {:id ::map-reduce 254 | :in {:in-1 ::s1 255 | :in-2 ::s2} 256 | :out {:out ::q1} 257 | :opts {:reduce-fn + 258 | :map-fn inc 259 | :to [:out]}} 260 | {:id ::done 261 | :in {:in ::q1} 262 | :opts {:done done 263 | :x-n 9}}] 264 | :queue-opts {::q1 {:queue-meta #{:q/t :tx/t}}}}] 265 | (q/send! g ::s1 {:x 1}) 266 | (q/send! g ::s1 {:x 3}) 267 | (q/send! g ::s2 {:x 2}) 268 | (q/send! g ::s2 {:x 4}) 269 | (is (done? done)) 270 | (is (= (q/all-graph-messages g) 271 | {::qt/error [] 272 | ::s1 [{:x 1 :q/meta {:q/queue {::s1 {:q/t 1}}}} 273 | {:x 3 :q/meta {:q/queue {::s1 {:q/t 2}}}}] 274 | ::s2 [{:x 2 :q/meta {:q/queue {::s2 {:q/t 1}}}} 275 | {:x 4 :q/meta {:q/queue {::s2 {:q/t 2}}}}] 276 | ::q1 [{:x 5 :q/meta {:tx/t 1 277 | :q/queue {::s1 {:q/t 1} 278 | ::s2 {:q/t 1} 279 | ::q1 {:q/t 1}}}} 280 | {:x 9 :q/meta {:tx/t 2 281 | :q/queue {::s1 {:q/t 2} 282 | ::s2 {:q/t 2} 283 | ::q1 {:q/t 2}}}}]}))))) 284 | 285 | (t/deftest graph-join-fork-test 286 | (let [q1-done (promise) 287 | q2-done (promise)] 288 | (qt/with-graph-and-delete 289 | [g {:id ::graph 290 | :processors [{:id ::s1} 291 | {:id ::s2} 292 | {:id ::map-reduce 293 | :opts {:reduce-fn + 294 | :to [:out-1 :out-2]} 295 | :in {:in-1 ::s1 296 | :in-2 ::s2} 297 | :out {:out-1 ::q1 298 | :out-2 ::q2}} 299 | {:id ::k1 300 | :fn ::done 301 | :in {:in ::q1} 302 | :opts {:done q1-done 303 | :x-n 7}} 304 | {:id ::k2 305 | :fn ::done 306 | :in {:in ::q2} 307 | :opts {:done q2-done 308 | :x-n 7}}] 309 | :queue-opts {::s1 {:queue-meta #{:q/t :tx/t}}}}] 310 | (q/send! g ::s1 {:x 1}) 311 | (q/send! g ::s1 {:x 3}) 312 | (q/send! g ::s2 {:x 2}) 313 | (q/send! g ::s2 {:x 4}) 314 | (is (and (done? q1-done) 315 | (done? q2-done))) 316 | (is (= (q/all-graph-messages g) 317 | {::qt/error [] 318 | ::s1 [{:x 1 :q/meta {:tx/t 1 :q/queue {::s1 {:q/t 1}}}} 319 | {:x 3 :q/meta {:tx/t 2 :q/queue {::s1 {:q/t 2}}}}] 320 | ::s2 [{:x 2 :q/meta {:q/queue {::s2 {:q/t 1}}}} 321 | {:x 4 :q/meta {:q/queue {::s2 {:q/t 2}}}}] 322 | ::q1 [{:x 3 :q/meta {:tx/t 1 323 | :q/queue {::s1 {:q/t 1} 324 | ::s2 {:q/t 1} 325 | ::q1 {:q/t 1}}}} 326 | {:x 7 :q/meta {:tx/t 2 327 | :q/queue {::s1 {:q/t 2} 328 | ::s2 {:q/t 2} 329 | ::q1 {:q/t 2}}}}] 330 | ::q2 [{:x 3 :q/meta {:tx/t 1 331 | :q/queue {::s1 {:q/t 1} 332 | ::s2 {:q/t 1} 333 | ::q2 {:q/t 1}}}} 334 | {:x 7 :q/meta {:tx/t 2 335 | :q/queue {::s1 {:q/t 2} 336 | ::s2 {:q/t 2} 337 | ::q2 {:q/t 2}}}}]}))))) 338 | 339 | (defmethod q/processor ::join-fork-conditional 340 | [_ msgs] 341 | (let [n (transduce (map :x) + (vals msgs)) 342 | msg {:x n}] 343 | {:even (when (even? n) msg) 344 | :odd (when (odd? n) msg)})) 345 | 346 | (t/deftest graph-join-fork-conditional-test 347 | (let [q1-done (promise) 348 | q2-done (promise)] 349 | (qt/with-graph-and-delete 350 | [g {:id ::graph 351 | :processors [{:id ::s1} 352 | {:id ::s2} 353 | {:id ::join-fork-conditional 354 | :in {:in-1 ::s1 355 | :in-2 ::s2} 356 | :out {:even ::q1 357 | :odd ::q2}} 358 | {:id ::k1 359 | :fn ::done 360 | :in {:in ::q1} 361 | :opts {:done q1-done 362 | :x-n 2}} 363 | {:id ::k2 364 | :fn ::done 365 | :in {:in ::q2} 366 | :opts {:done q2-done 367 | :x-n 5}}] 368 | :queue-opts {::s1 {:queue-meta #{:q/t :tx/t}}}}] 369 | (q/send! g ::s1 {:x 1}) 370 | (q/send! g ::s1 {:x 2}) 371 | (q/send! g ::s2 {:x 1}) 372 | (q/send! g ::s2 {:x 3}) 373 | (is (and (done? q1-done) 374 | (done? q2-done))) 375 | (is (= (q/all-graph-messages g) 376 | {::qt/error [] 377 | ::s1 [{:x 1 :q/meta {:tx/t 1 :q/queue {::s1 {:q/t 1}}}} 378 | {:x 2 :q/meta {:tx/t 2 :q/queue {::s1 {:q/t 2}}}}] 379 | ::s2 [{:x 1 :q/meta {:q/queue {::s2 {:q/t 1}}}} 380 | {:x 3 :q/meta {:q/queue {::s2 {:q/t 2}}}}] 381 | ::q1 [{:x 2 :q/meta {:tx/t 1 382 | :q/queue {::s1 {:q/t 1} 383 | ::s2 {:q/t 1} 384 | ::q1 {:q/t 1}}}}] 385 | ::q2 [{:x 5 :q/meta {:tx/t 2 386 | :q/queue {::s1 {:q/t 2} 387 | ::s2 {:q/t 2} 388 | ::q2 {:q/t 1}}}}]}))))) 389 | 390 | (defmethod q/processor ::pipe-error 391 | [_ {msg :in}] 392 | (if (even? (:x msg)) 393 | {:out msg} 394 | (throw (Exception. "Oops")))) 395 | 396 | (t/deftest graph-error-test 397 | ;; Suppress exception logging, just testing messaging 398 | (log/with-level :fatal 399 | (let [done (promise) 400 | done-errors (promise)] 401 | (qt/with-graph-and-delete 402 | [g {:id ::graph 403 | :processors [{:id ::s1} 404 | {:id ::pipe-error 405 | :in {:in ::s1} 406 | :out {:out ::q1}} 407 | {:id ::d1 408 | :fn ::done 409 | :in {:in ::q1} 410 | :opts {:done done 411 | :x-n 4}} 412 | {:id ::d2 413 | :fn ::done-counter 414 | :in {:in ::qt/error} 415 | :opts {:done done-errors 416 | :counter (atom 0) 417 | :n 2}}]}] 418 | (q/send! g ::s1 {:x 1}) 419 | (q/send! g ::s1 {:x 2}) 420 | (q/send! g ::s1 {:x 3}) 421 | (q/send! g ::s1 {:x 4}) 422 | (is (done? done)) 423 | (is (done? done-errors)) 424 | (is (= (-> (q/all-graph-messages g) 425 | (update ::qt/error qt/simplify-exceptions)) 426 | {::qt/error [{:q/type :q.type.err/processor 427 | :err/cause {:cause "Oops"} 428 | :err.proc/config {:id ::pipe-error 429 | :in {:in ::s1} 430 | :out {:out ::q1} 431 | :strategy ::q/exactly-once} 432 | :err.proc/messages {::s1 {:x 1 433 | :q/meta {:q/queue {::s1 {:q/t 1}}}}} 434 | :q/meta {:q/queue {::qt/error {:q/t 1}}}} 435 | {:q/type :q.type.err/processor 436 | :err/cause {:cause "Oops"} 437 | :err.proc/config {:id ::pipe-error 438 | :in {:in ::s1} 439 | :out {:out ::q1} 440 | :strategy ::q/exactly-once} 441 | :err.proc/messages {::s1 {:x 3 442 | :q/meta {:q/queue {::s1 {:q/t 3}}}}} 443 | :q/meta {:q/queue {::qt/error {:q/t 2}}}}] 444 | ::s1 [{:x 1 :q/meta {:q/queue {::s1 {:q/t 1}}}} 445 | {:x 2 :q/meta {:q/queue {::s1 {:q/t 2}}}} 446 | {:x 3 :q/meta {:q/queue {::s1 {:q/t 3}}}} 447 | {:x 4 :q/meta {:q/queue {::s1 {:q/t 4}}}}] 448 | ::q1 [{:x 2 :q/meta {:q/queue {::s1 {:q/t 2} 449 | ::q1 {:q/t 1}}}} 450 | {:x 4 :q/meta {:q/queue {::s1 {:q/t 4} 451 | ::q1 {:q/t 2}}}}]})))))) 452 | 453 | (t/deftest filter-messages-test 454 | (let [msg {::q {:q/topics {::t1 ::message 455 | ::t2 ::message}}}] 456 | (is (= (#'q/topics-filter nil nil))) 457 | (is (= (#'q/topics-filter nil {}))) 458 | (is (= (#'q/topics-filter [] nil))) 459 | (is (= (#'q/topics-filter [::t1 ::t2] msg) 460 | msg)) 461 | (is (= (#'q/topics-filter [::t1] msg) 462 | {::q {:q/topics {::t1 ::message 463 | ::t2 ::message}}})) 464 | (is (= (#'q/topics-filter [::other] msg) 465 | nil)) 466 | (is (= (#'q/topics-filter [::t1 ::t2] 467 | {::q1 {:q/topics {::t1 ::message 468 | ::t2 ::message}} 469 | ::q2 {:q/topics {::t1 ::message 470 | ::t2 ::message}}}) 471 | {::q1 {:q/topics {::t1 ::message 472 | ::t2 ::message}} 473 | ::q2 {:q/topics {::t1 ::message 474 | ::t2 ::message}}})) 475 | (is (= (#'q/topics-filter [::t1] 476 | {::q1 {:q/topics {::t1 ::message 477 | ::t2 ::message}} 478 | ::q2 {:q/topics {::t1 ::message 479 | ::t2 ::message}}}) 480 | {::q1 {:q/topics {::t1 ::message 481 | ::t2 ::message}} 482 | ::q2 {:q/topics {::t1 ::message 483 | ::t2 ::message}}})) 484 | (is (= (#'q/topics-filter [::t2] 485 | {::q1 {:q/topics {::t2 ::message}} 486 | ::q2 {:q/topics {::t1 ::message}}}) 487 | {::q1 {:q/topics {::t2 ::message}}})) 488 | (is (= (#'q/topics-filter [::t1] 489 | {::q1 {:q/topics {::t2 ::message}} 490 | ::q2 {:q/topics {::t1 ::message}}}) 491 | {::q2 {:q/topics {::t1 ::message}}})) 492 | (is (= (#'q/topics-filter [::t2] 493 | {::q1 {:q/topics {::t1 ::message}} 494 | ::q2 {:q/topics {::t1 ::message}}}) 495 | nil)))) 496 | 497 | (defmethod q/processor ::processor-a 498 | [{:keys [system]} {msg :in}] 499 | {:out (assoc msg 500 | :processed true 501 | :system system)}) 502 | 503 | (t/deftest processor-fn-test 504 | (let [config {:id ::processor-a 505 | :topics [::doc] 506 | :in {:in ::q1 507 | :in-ignored ::q2} 508 | :out {:out ::tx}} 509 | f (#'q/get-processor-fn {:config config}) 510 | process {:config config 511 | :system {:component true}}] 512 | (is (fn? f)) 513 | (is (nil? (f process nil))) 514 | (is (nil? (f process {}))) 515 | (is (nil? (f process {::q1 {}}))) 516 | (is (nil? (f process {::q1 {:q/topics nil}}))) 517 | (is (nil? (f process {::q1 {:q/topics false}}))) 518 | (is (nil? (f process {::q1 {:q/topics {}}}))) 519 | (is (nil? (f process {::q1 {:q/topics {::other ::message}}}))) 520 | (is (= (f process {::q1 {:q/topics {::doc ::message}}}) 521 | {::tx {:q/topics {::doc ::message} 522 | :processed true 523 | :system {:component true}}})))) 524 | 525 | (defn- process-b 526 | [doc] 527 | (assoc doc ::processed true)) 528 | 529 | (defmethod q/processor ::processor-b 530 | [_ {msg :in 531 | _ :in-ignored}] 532 | {:out (->> process-b 533 | (partial cutil/map-vals) 534 | (update msg :q/topics))}) 535 | 536 | (defmethod q/processor ::doc-store 537 | [{{db :db 538 | done :done} :opts} 539 | {{{{id :system/uuid 540 | :as doc} ::doc} :q/topics 541 | {t :tx/t} :q/meta 542 | :as msg} :in}] 543 | (swap! db update id merge doc) 544 | (when (= t 2) 545 | (deliver done true))) 546 | 547 | (defn message 548 | [topics] 549 | {:q/type :q.type.tx/command 550 | :q/topics topics}) 551 | 552 | (t/deftest graph-filters-test 553 | (let [done (promise) 554 | db (atom {})] 555 | (qt/with-graph-and-delete 556 | [g {:id ::graph 557 | :queue-opts {::tx {:queue-meta #{:q/t :tx/t}}} 558 | :processors [{:id ::s1} 559 | {:id ::s2} 560 | {:id ::processor-b 561 | :types :q.type.tx/command 562 | :in {:in ::s1 563 | :in-ignored ::s2} 564 | :out {:out ::tx}} 565 | {:id ::doc-store 566 | :topics ::doc 567 | :in {:in ::tx} 568 | :opts {:db db 569 | :done done}}]}] 570 | (q/send! g ::s1 (message {::other {:system/uuid 1 :text "no dice"}})) 571 | (q/send! g ::s2 (message {::other {:system/uuid 2 :text "no dice"}})) 572 | (q/send! g ::s1 (message {::doc {:system/uuid 3 :text "text"}})) 573 | (q/send! g ::s2 (message {::other {:system/uuid 4 :text "no dice"}})) 574 | (is (done? done)) 575 | (is (= @db 576 | {3 {:system/uuid 3 577 | ::processed true 578 | :text "text"}})) 579 | (is (= (q/all-graph-messages g) 580 | {::qt/error [] 581 | ::s1 [{:q/type :q.type.tx/command 582 | :q/topics {::other {:system/uuid 1 :text "no dice"}} 583 | :q/meta {:q/queue {::s1 {:q/t 1}}}} 584 | {:q/type :q.type.tx/command 585 | :q/topics {::doc {:system/uuid 3 :text "text"}} 586 | :q/meta {:q/queue {::s1 {:q/t 2}}}}] 587 | ::s2 [{:q/type :q.type.tx/command 588 | :q/topics {::other {:system/uuid 2 :text "no dice"}} 589 | :q/meta {:q/queue {::s2 {:q/t 1}}}} 590 | {:q/type :q.type.tx/command 591 | :q/topics {::other {:system/uuid 4 :text "no dice"}} 592 | :q/meta {:q/queue {::s2 {:q/t 2}}}}] 593 | ::tx [{:q/type :q.type.tx/command 594 | :q/topics {::other {:system/uuid 1 :text "no dice" 595 | ::processed true}} 596 | :q/meta {:q/queue {::s1 {:q/t 1} 597 | ::s2 {:q/t 1} 598 | ::tx {:q/t 1}} 599 | :tx/t 1}} 600 | {:q/type :q.type.tx/command 601 | :q/topics {::doc {:system/uuid 3 :text "text" 602 | ::processed true}} 603 | :q/meta {:q/queue {::s1 {:q/t 2} 604 | ::s2 {:q/t 2} 605 | ::tx {:q/t 2}} 606 | :tx/t 2}}]}))))) 607 | 608 | ;; The tests beyond this point deal with implementation details of 609 | ;; persistence and message delivery semantics. Specifically they lift 610 | ;; processor backing queues into the graph so that they can be 611 | ;; inspected and managed concurrently. For this reason they should not 612 | ;; be taken as examples of normal usage. 613 | 614 | (defmethod q/processor ::interrupt 615 | [{{:keys [done interrupt?]} :opts} {msg :in}] 616 | (when interrupt? 617 | (try 618 | (throw-interrupt!) 619 | (finally (deliver done true)))) 620 | {:out msg}) 621 | 622 | (defn try-messages 623 | [g] 624 | (->> (:processors g) 625 | (vals) 626 | (mapcat q/get-try-queues) 627 | (map (juxt :id q/all-messages)) 628 | (into {}))) 629 | 630 | (t/deftest exactly-once-processor-unhandled-interrupt-test 631 | ;; The processor :fn throws an unhandled interrupt during the method 632 | ;; execution. The processor quits due to the unhandled 633 | ;; interrupt. Without exactly once semantics the first message would 634 | ;; be lost. However, using exactly once semantics the message is 635 | ;; persisted after the graph is restarted. 636 | (let [p-id :cues.queue-test.graph.cues.queue-test/interrupt 637 | p-tid :cues.queue-test.graph.cues.queue-test.interrupt.cues.queue-test/s1 638 | d-id :cues.queue-test.graph.cues.queue-test/done 639 | d-tid :cues.queue-test.graph.cues.queue-test.done.cues.queue-test/q1] 640 | (let [done-1 (promise) 641 | done-2 (promise) 642 | g (->> {:id ::graph 643 | :processors [{:id ::s1} 644 | {:id ::interrupt 645 | :in {:in ::s1} 646 | :out {:out ::q1} 647 | :opts {:interrupt? true 648 | :done done-1}} 649 | {:id ::d2 650 | :fn ::done-counter 651 | :in {:in p-id} 652 | :opts {:done done-2 653 | :counter (atom 0) 654 | :n 1}}]} 655 | (q/graph) 656 | (q/start-graph!))] 657 | (q/send! g ::s1 {:x 1}) 658 | (q/send! g ::s1 {:x 2}) 659 | (is (done? done-1)) 660 | (is (done? done-2)) 661 | (is (= (q/all-graph-messages g) 662 | {::s1 [{:x 1} {:x 2}] 663 | ::q1 [] 664 | p-id [{:q/type :q.type/snapshot 665 | :q/proc-id p-id 666 | :q/tailer-indices {p-tid 1}}]})) 667 | (q/stop-graph! g) 668 | (q/close-graph! g)) 669 | 670 | ;; After restarting the graph both messages are delivered, and we 671 | ;; can see the subsequent attempt in the backing queue. 672 | (let [done-1 (promise) 673 | done-2 (promise) 674 | done-3 (promise) 675 | g (->> {:id ::graph 676 | :processors [{:id ::s1} 677 | {:id ::interrupt 678 | :in {:in ::s1} 679 | :out {:out ::q1}} 680 | {:id ::done 681 | :in {:in ::q1} 682 | :opts {:done done-1 683 | :x-n 2}} 684 | {:id ::d2 685 | :fn ::done-counter 686 | :in {:in p-id} 687 | :opts {:done done-2 688 | :counter (atom 0) 689 | :n 4}} ; 4 + 1 from previous graph 690 | {:id ::d3 691 | :fn ::done-counter 692 | :in {:in d-id} 693 | :opts {:done done-3 694 | :counter (atom 0) 695 | :n 5}}]} 696 | (q/graph) 697 | (q/start-graph!))] 698 | (is (done? done-1)) 699 | (is (done? done-2)) 700 | (is (done? done-3)) 701 | (is (= (q/all-graph-messages g) 702 | {::s1 [{:x 1} {:x 2}] 703 | ::q1 [{:x 1} {:x 2}] 704 | p-id [{:q/type :q.type/snapshot 705 | :q/proc-id p-id 706 | :q/tailer-indices {p-tid 1}} 707 | {:q/type :q.type/attempt-output 708 | :q/message-index 1} 709 | {:q/type :q.type/snapshot 710 | :q/proc-id p-id 711 | :q/tailer-indices {p-tid 2}} 712 | {:q/type :q.type/attempt-output 713 | :q/message-index 2} 714 | {:q/type :q.type/snapshot 715 | :q/proc-id p-id 716 | :q/tailer-indices {p-tid 3}}] 717 | d-id [{:q/type :q.type/snapshot 718 | :q/proc-id d-id 719 | :q/tailer-indices {d-tid 1}} 720 | {:q/type :q.type/attempt-nil 721 | :q/hash -1173148930} 722 | {:q/type :q.type/snapshot 723 | :q/proc-id d-id 724 | :q/tailer-indices {d-tid 2}} 725 | {:q/type :q.type/attempt-nil 726 | :q/hash 1121832113} 727 | {:q/type :q.type/snapshot 728 | :q/proc-id d-id 729 | :q/tailer-indices {d-tid 3}}]})) 730 | (q/stop-graph! g) 731 | (q/close-and-delete-graph! g true)))) 732 | 733 | (defmethod q/processor ::interrupt-alts 734 | [{{:keys [done interrupt?]} :opts} msgs] 735 | (when interrupt? 736 | (try 737 | (throw-interrupt!) 738 | (finally (deliver done true)))) 739 | {:out (first (vals msgs))}) 740 | 741 | (t/deftest exactly-once-processor-unhandled-interrupt-alts-test 742 | ;; The alts processor :fn throws an unhandled interrupt during the 743 | ;; method execution. The processor quits due to the unhandled 744 | ;; interrupt. Without exactly once semantics the first message would 745 | ;; be lost. However, using exactly once semantics the message is 746 | ;; persisted after the graph is restarted. 747 | (let [p-id :cues.queue-test.graph.cues.queue-test/interrupt-alts 748 | p-tid1 :cues.queue-test.graph.cues.queue-test.interrupt-alts.cues.queue-test/s1 749 | p-tid2 :cues.queue-test.graph.cues.queue-test.interrupt-alts.cues.queue-test/s2 750 | d-id :cues.queue-test.graph.cues.queue-test/done 751 | d-tid :cues.queue-test.graph.cues.queue-test.done.cues.queue-test/q1] 752 | (let [done-1 (promise) 753 | done-2 (promise) 754 | g (->> {:id ::graph 755 | :processors [{:id ::s1} 756 | {:id ::s2} 757 | {:id ::interrupt-alts 758 | :alts {:in-1 ::s1 759 | :in-2 ::s2} 760 | :out {:out ::q1} 761 | :opts {:interrupt? true 762 | :done done-1}} 763 | {:id ::d2 764 | :fn ::done-counter 765 | :in {:in p-id} 766 | :opts {:done done-2 767 | :counter (atom 0) 768 | :n 2}}]} 769 | (q/graph) 770 | (q/start-graph!))] 771 | (q/send! g ::s2 {:x 1}) 772 | (is (done? done-1)) 773 | (q/send! g ::s1 {:x 2}) 774 | (is (done? done-2)) 775 | (is (= (q/all-graph-messages g) 776 | {::s1 [{:x 2}] 777 | ::s2 [{:x 1}] 778 | ::q1 [] 779 | p-id [{:q/type :q.type/snapshot 780 | :q/proc-id p-id 781 | :q/tailer-indices {p-tid1 1 p-tid2 1}} 782 | {:q/type :q.type/snapshot-alts 783 | :q/tailer-id p-tid2}]})) 784 | (q/stop-graph! g) 785 | (q/close-graph! g)) 786 | 787 | ;; After restarting the graph both messages are delivered, and we 788 | ;; can see the subsequent attempt in the backing queue. 789 | (let [done-1 (promise) 790 | done-2 (promise) 791 | done-3 (promise) 792 | g (->> {:id ::graph 793 | :processors [{:id ::s1} 794 | {:id ::s2} 795 | {:id ::interrupt-alts 796 | :alts {:in-2 ::s2 797 | :in-1 ::s1} 798 | :out {:out ::q1}} 799 | {:id ::done 800 | :in {:in ::q1} 801 | :opts {:done done-1 802 | :x-n 2}} 803 | {:id ::d2 804 | :fn ::done-counter 805 | :in {:in p-id} 806 | :opts {:done done-2 807 | :counter (atom 0) 808 | :n 5}} ; 5 + 2 from previous graph 809 | {:id ::d3 810 | :fn ::done-counter 811 | :in {:in d-id} 812 | :opts {:done done-3 813 | :counter (atom 0) 814 | :n 5}}]} 815 | (q/graph) 816 | (q/start-graph!))] 817 | (is (done? done-1)) 818 | (is (done? done-2)) 819 | (is (done? done-3)) 820 | (is (= (q/all-graph-messages g) 821 | {::s1 [{:x 2}] 822 | ::s2 [{:x 1}] 823 | ::q1 [{:x 1} 824 | {:x 2}] 825 | p-id [{:q/type :q.type/snapshot 826 | :q/proc-id p-id 827 | :q/tailer-indices {p-tid1 1 p-tid2 1}} 828 | {:q/type :q.type/snapshot-alts 829 | :q/tailer-id p-tid2} 830 | {:q/type :q.type/attempt-output 831 | :q/message-index 1} 832 | {:q/type :q.type/snapshot 833 | :q/proc-id p-id 834 | :q/tailer-indices {p-tid1 1 p-tid2 2}} 835 | {:q/type :q.type/snapshot-alts 836 | :q/tailer-id p-tid1} 837 | {:q/type :q.type/attempt-output 838 | :q/message-index 2} 839 | {:q/type :q.type/snapshot 840 | :q/proc-id p-id 841 | :q/tailer-indices {p-tid1 2 p-tid2 2}}] 842 | d-id [{:q/type :q.type/snapshot 843 | :q/proc-id d-id 844 | :q/tailer-indices {d-tid 1}} 845 | {:q/type :q.type/attempt-nil 846 | :q/hash -1173148930} 847 | {:q/type :q.type/snapshot 848 | :q/proc-id d-id 849 | :q/tailer-indices {d-tid 2}} 850 | {:q/type :q.type/attempt-nil 851 | :q/hash 1121832113} 852 | {:q/type :q.type/snapshot 853 | :q/proc-id d-id 854 | :q/tailer-indices {d-tid 3}}]})) 855 | (q/stop-graph! g) 856 | (q/close-and-delete-graph! g true)))) 857 | 858 | (defmethod q/processor ::interrupt-fork 859 | [{{:keys [done interrupt?]} :opts} {:keys [in-1 in-2]}] 860 | (when interrupt? 861 | (try 862 | (throw-interrupt!) 863 | (finally (deliver done true)))) 864 | {:out-1 in-1 865 | :out-2 in-2}) 866 | 867 | (t/deftest exactly-once-join-fork-unhandled-interrupt-test 868 | ;; Same as exactly-once-processor-unhandled-interrupt-test, but 869 | ;; tests exactly once delivery on join-fork intermediary processing 870 | ;; loops. 871 | (let [p-id :cues.queue-test.graph.cues.queue-test/interrupt-fork 872 | p-tid1 :cues.queue-test.graph.cues.queue-test.interrupt-fork.cues.queue-test/s1 873 | p-tid2 :cues.queue-test.graph.cues.queue-test.interrupt-fork.cues.queue-test/s2 874 | p-jid1 :cues.queue-test.graph.cues.queue-test.interrupt-fork.cues.queue-test/q1 875 | p-jid2 :cues.queue-test.graph.cues.queue-test.interrupt-fork.cues.queue-test/q2 876 | p-fid1 :cues.queue-test.graph.cues.queue-test.interrupt-fork.cues.queue-test.q1.cues.queue-test.graph.cues.queue-test.interrupt-fork/fork 877 | p-fid2 :cues.queue-test.graph.cues.queue-test.interrupt-fork.cues.queue-test.q2.cues.queue-test.graph.cues.queue-test.interrupt-fork/fork] 878 | (let [done-1 (promise) 879 | done-2 (promise) 880 | done-3 (promise) 881 | done-4 (promise) 882 | g (->> {:id ::graph 883 | :processors [{:id ::s1} 884 | {:id ::s2} 885 | {:id ::interrupt-fork 886 | :in {:in-1 ::s1 887 | :in-2 ::s2} 888 | :out {:out-1 ::q1 889 | :out-2 ::q2} 890 | :opts {:interrupt? true 891 | :done done-1}} 892 | {:id ::d2 893 | :fn ::done-counter 894 | :in {:in p-id} 895 | :opts {:done done-2 896 | :counter (atom 0) 897 | :n 1}} 898 | {:id ::d3 899 | :fn ::done-counter 900 | :in {:in p-jid1} 901 | :opts {:done done-3 902 | :counter (atom 0) 903 | :n 1}} 904 | {:id ::d4 905 | :fn ::done-counter 906 | :in {:in p-jid2} 907 | :opts {:done done-4 908 | :counter (atom 0) 909 | :n 1}}]} 910 | (q/graph) 911 | (q/start-graph!))] 912 | (q/send! g ::s1 {:x 1}) 913 | (q/send! g ::s2 {:x 2}) 914 | (is (done? done-1)) 915 | (is (done? done-2)) 916 | (is (done? done-3)) 917 | (is (done? done-4)) 918 | (is (= (q/all-graph-messages g) 919 | {::s1 [{:x 1}] 920 | ::s2 [{:x 2}] 921 | ::q1 [] 922 | ::q2 [] 923 | p-id [{:q/type :q.type/snapshot 924 | :q/proc-id p-id 925 | :q/tailer-indices {p-tid1 1 p-tid2 1}}] 926 | p-jid1 [{:q/type :q.type/snapshot 927 | :q/proc-id p-jid1 928 | :q/tailer-indices {p-fid1 1}}] 929 | p-jid2 [{:q/type :q.type/snapshot 930 | :q/proc-id p-jid2 931 | :q/tailer-indices {p-fid2 1}}]})) 932 | (q/stop-graph! g) 933 | (q/close-graph! g)) 934 | 935 | ;; After restarting the graph both messages are delivered, and we 936 | ;; can see the subsequent attempt in the backing queue. 937 | (let [done-1 (promise) 938 | done-2 (promise) 939 | done-3 (promise) 940 | done-4 (promise) 941 | g (->> {:id ::graph 942 | :processors [{:id ::s1} 943 | {:id ::s2} 944 | {:id ::interrupt-fork 945 | :in {:in-1 ::s1 946 | :in-2 ::s2} 947 | :out {:out-1 ::q1 948 | :out-2 ::q2}} 949 | {:id ::done 950 | :in {:in ::q1} 951 | :opts {:done done-1 952 | :x-n 1}} 953 | {:id ::d2 954 | :fn ::done-counter 955 | :in {:in p-id} 956 | :opts {:done done-2 957 | :counter (atom 0) 958 | :n 2}} ; 2 + 1 from intermediary graph 959 | {:id ::d3 960 | :fn ::done-counter 961 | :in {:in p-jid1} 962 | :opts {:done done-3 963 | :counter (atom 0) 964 | :n 2}} ; 2 + 1 from intermediary graph 965 | {:id ::d4 966 | :fn ::done-counter 967 | :in {:in p-jid2} 968 | :opts {:done done-4 969 | :counter (atom 0) 970 | :n 2}}]} ; 2 + 1 from intermediary graph 971 | (q/graph) 972 | (q/start-graph!))] 973 | (is (done? done-1)) 974 | (is (done? done-2)) 975 | (is (done? done-3)) 976 | (is (done? done-4)) 977 | (is (= (q/all-graph-messages g) 978 | {::s1 [{:x 1}] 979 | ::s2 [{:x 2}] 980 | ::q1 [{:x 1}] 981 | ::q2 [{:x 2}] 982 | p-id [{:q/type :q.type/snapshot 983 | :q/proc-id p-id 984 | :q/tailer-indices {p-tid1 1 p-tid2 1}} 985 | {:q/type :q.type/attempt-output 986 | :q/message-index 1} 987 | {:q/type :q.type/snapshot 988 | :q/proc-id p-id 989 | :q/tailer-indices {p-tid1 2 p-tid2 2}}] 990 | p-jid1 [{:q/type :q.type/snapshot 991 | :q/proc-id p-jid1 992 | :q/tailer-indices {p-fid1 1}} 993 | {:q/type :q.type/attempt-output 994 | :q/message-index 1} 995 | {:q/type :q.type/snapshot 996 | :q/proc-id p-jid1 997 | :q/tailer-indices {p-fid1 2}}] 998 | p-jid2 [{:q/type :q.type/snapshot 999 | :q/proc-id p-jid2 1000 | :q/tailer-indices {p-fid2 1}} 1001 | {:q/type :q.type/attempt-output 1002 | :q/message-index 1} 1003 | {:q/type :q.type/snapshot 1004 | :q/proc-id p-jid2 1005 | :q/tailer-indices {p-fid2 2}}]})) 1006 | (q/stop-graph! g) 1007 | (q/close-and-delete-graph! g true)))) 1008 | 1009 | (defmethod q/processor ::exception 1010 | ;; Throws only first message. 1011 | [{{:keys [counter n]} :opts} {msg :in}] 1012 | (err/wrap-error {:err/context "context"} 1013 | (if (= (swap! counter inc) n) 1014 | (throw (Exception. "Oops")) 1015 | {:out msg}))) 1016 | 1017 | (t/deftest exactly-once-processor-handled-error-test 1018 | ;; The processor throws a handled error only on the first 1019 | ;; message. Handled errors are considered delivered according 1020 | ;; exactly once semantics. The error queue is configured and the 1021 | ;; error should appear there. 1022 | (log/with-level :fatal 1023 | (let [p-id :cues.queue-test.graph.cues.queue-test/exception 1024 | p-tid :cues.queue-test.graph.cues.queue-test.exception.cues.queue-test/s1 1025 | d-id :cues.queue-test.graph.cues.queue-test/done 1026 | d-tid :cues.queue-test.graph.cues.queue-test.done.cues.queue-test/q1 1027 | done-1 (promise) 1028 | done-2 (promise) 1029 | done-3 (promise)] 1030 | (qt/with-graph-and-delete 1031 | [g {:id ::graph 1032 | :queue-opts {::q/default {:queue-meta false}} 1033 | :processors [{:id ::s1} 1034 | {:id ::exception 1035 | :in {:in ::s1} 1036 | :out {:out ::q1} 1037 | :opts {:counter (atom 0) 1038 | :n 1}} 1039 | {:id ::done 1040 | :in {:in ::q1} 1041 | :opts {:done done-1 1042 | :x-n 2}} 1043 | {:id ::d2 1044 | :fn ::done-counter 1045 | :in {:in p-id} 1046 | :opts {:done done-2 1047 | :counter (atom 0) 1048 | :n 5}} 1049 | {:id ::d3 1050 | :fn ::done-counter 1051 | :in {:in d-id} 1052 | :opts {:done done-3 1053 | :counter (atom 0) 1054 | :n 3}}]}] 1055 | (q/send! g ::s1 {:x 1}) 1056 | (q/send! g ::s1 {:x 2}) 1057 | (is (done? done-1)) 1058 | (is (done? done-2)) 1059 | (is (done? done-3)) 1060 | (is (= (-> (q/all-graph-messages g) 1061 | (update ::qt/error qt/simplify-exceptions)) 1062 | {::qt/error [{:q/type :q.type.err/error 1063 | :err.proc/config {:id ::exception 1064 | :in {:in ::s1} 1065 | :out {:out ::q1} 1066 | :strategy ::q/exactly-once} 1067 | :err.proc/messages {::s1 {:x 1}} 1068 | :err/context "context" 1069 | :err/cause {:cause "Oops"}}] 1070 | ::s1 [{:x 1} {:x 2}] 1071 | ::q1 [{:x 2}] 1072 | p-id [{:q/type :q.type/snapshot 1073 | :q/proc-id p-id 1074 | :q/tailer-indices {p-tid 1}} 1075 | {:q/type :q.type/attempt-error 1076 | :q/message-index 1} 1077 | {:q/type :q.type/snapshot 1078 | :q/proc-id p-id 1079 | :q/tailer-indices {p-tid 2}} 1080 | {:q/type :q.type/attempt-output 1081 | :q/message-index 1} 1082 | {:q/type :q.type/snapshot 1083 | :q/proc-id p-id 1084 | :q/tailer-indices {p-tid 3}}] 1085 | d-id [{:q/type :q.type/snapshot 1086 | :q/proc-id d-id 1087 | :q/tailer-indices {d-tid 1}} 1088 | {:q/type :q.type/attempt-nil 1089 | :q/hash -1173148930} 1090 | {:q/type :q.type/snapshot 1091 | :q/proc-id d-id 1092 | :q/tailer-indices {d-tid 2}}] })))))) 1093 | 1094 | (def ^:private write-impl 1095 | "Bind original here for use in test fn." 1096 | q/write) 1097 | 1098 | (defn- write-error 1099 | "Throws an error on a specific attempt number." 1100 | [done counter n] 1101 | (fn [appender msg] 1102 | (write-impl appender msg) 1103 | (when (= (swap! counter inc) n) 1104 | (deliver done true)) 1105 | (when (= (:q/type msg) :q.type/attempt-output) 1106 | (throw (Exception. "Failed after write"))))) 1107 | 1108 | (t/deftest exactly-once-write-handled-error-test 1109 | ;; The processor throws a handled error after the attempt is 1110 | ;; persisted, but before the output message is written. The error 1111 | ;; queue is not configured, so message is considered "handled" after 1112 | ;; logging output. 1113 | (log/with-level :fatal 1114 | (let [p-id :cues.queue-test.graph.cues.queue-test/map-reduce 1115 | p-tid :cues.queue-test.graph.cues.queue-test.map-reduce.cues.queue-test/s1] 1116 | (let [done-1 (promise) 1117 | done-2 (promise) 1118 | logged (atom 0)] 1119 | (with-redefs [q/write (write-error done-1 (atom 0) 9) 1120 | q/log-processor-error (fn [_ _] (swap! logged inc))] 1121 | (let [g (->> {:id ::graph 1122 | :errors ::errors 1123 | :processors [{:id ::s1} 1124 | {:id ::map-reduce 1125 | :in {:in ::s1} 1126 | :out {:out ::q1} 1127 | :opts {:map-fn inc 1128 | :to [:out]}} 1129 | {:id ::done-counter 1130 | :in {:in p-id} 1131 | :opts {:done done-2 1132 | :counter (atom 0) 1133 | :n 7}}]} 1134 | (q/graph) 1135 | (q/start-graph!))] 1136 | (q/send! g ::s1 {:x 1}) 1137 | (q/send! g ::s1 {:x 2}) 1138 | (is (done? done-1)) 1139 | (is (done? done-2)) 1140 | (is (= @logged 2)) 1141 | (is (= (-> (q/all-graph-messages g) 1142 | (update ::errors qt/simplify-exceptions)) 1143 | {::errors [{:q/type :q.type.err/processor 1144 | :err/cause {:cause "Failed after write"} 1145 | :err.proc/config {:id ::map-reduce 1146 | :in {:in ::s1} 1147 | :out {:out ::q1} 1148 | :strategy ::q/exactly-once} 1149 | :err.proc/messages {:x 2}} 1150 | {:q/type :q.type.err/processor 1151 | :err/cause {:cause "Failed after write"} 1152 | :err.proc/config {:id ::map-reduce 1153 | :in {:in ::s1} 1154 | :out {:out ::q1} 1155 | :strategy ::q/exactly-once} 1156 | :err.proc/messages {:x 3}}] 1157 | ::s1 [{:x 1} {:x 2}] 1158 | ::q1 [] 1159 | p-id [{:q/type :q.type/snapshot 1160 | :q/proc-id p-id 1161 | :q/tailer-indices {p-tid 1}} 1162 | {:q/type :q.type/attempt-output 1163 | :q/message-index 1} 1164 | {:q/type :q.type/attempt-error 1165 | :q/message-index 1} 1166 | {:q/type :q.type/snapshot 1167 | :q/proc-id p-id 1168 | :q/tailer-indices {p-tid 2}} 1169 | {:q/type :q.type/attempt-output 1170 | :q/message-index 1} 1171 | {:q/type :q.type/attempt-error 1172 | :q/message-index 2} 1173 | {:q/type :q.type/snapshot 1174 | :q/proc-id p-id 1175 | :q/tailer-indices {p-tid 3}}]})) 1176 | (q/stop-graph! g) 1177 | (q/close-and-delete-graph! g true))))))) 1178 | 1179 | (t/deftest exactly-once-attempt-no-write-unhandled-error-test 1180 | ;; The processor throws an unhandled error after the attempt is 1181 | ;; persisted, but before the output message is written. The error 1182 | ;; queue is not configured so the error will not be handled. Without 1183 | ;; exactly once semantics the emssages would be lost. 1184 | (log/with-level :fatal 1185 | (let [p-id :cues.queue-test.graph.cues.queue-test/map-reduce 1186 | p-tid :cues.queue-test.graph.cues.queue-test.map-reduce.cues.queue-test/s1 1187 | d-id :cues.queue-test.graph.cues.queue-test/done 1188 | d-tid :cues.queue-test.graph.cues.queue-test.done.cues.queue-test/q1] 1189 | (let [done-1 (promise) 1190 | done-2 (promise) 1191 | logged (atom 0)] 1192 | (with-redefs [q/write (write-error done-1 (atom 0) 9) 1193 | q/log-processor-error (fn [_ _] (swap! logged inc))] 1194 | (let [g (->> {:id ::graph 1195 | :processors [{:id ::s1} 1196 | {:id ::map-reduce 1197 | :in {:in ::s1} 1198 | :out {:out ::q1} 1199 | :opts {:map-fn inc 1200 | :to [:out]}} 1201 | {:id ::done-counter 1202 | :in {:in p-id} 1203 | :opts {:done done-2 1204 | :counter (atom 0) 1205 | :n 2}}]} 1206 | (q/graph) 1207 | (q/start-graph!))] 1208 | (q/send! g ::s1 {:x 1}) 1209 | (q/send! g ::s1 {:x 2}) 1210 | (is (done? done-1)) 1211 | (is (done? done-2)) 1212 | (is (= @logged 1)) 1213 | (is (= (q/all-graph-messages g) 1214 | {::s1 [{:x 1} {:x 2}] 1215 | ::q1 [] 1216 | p-id [{:q/type :q.type/snapshot 1217 | :q/proc-id p-id 1218 | :q/tailer-indices {p-tid 1}} 1219 | {:q/type :q.type/attempt-output 1220 | :q/message-index 1}]})) 1221 | (q/stop-graph! g) 1222 | (q/close-graph! g)))) 1223 | 1224 | ;; After restarting the graph both messages are delivered, and we 1225 | ;; can see the subsequent attempt in the backing queue. 1226 | (let [done-2 (promise) 1227 | done-3 (promise) 1228 | logged (atom 0)] 1229 | (with-redefs [q/log-processor-error (fn [_ _] (swap! logged inc))] 1230 | (let [g (->> {:id ::graph 1231 | :processors [{:id ::s1} 1232 | {:id ::map-reduce 1233 | :in {:in ::s1} 1234 | :out {:out ::q1} 1235 | :opts {:map-fn inc 1236 | :to [:out]}} 1237 | {:id ::done 1238 | :in {:in ::q1}} 1239 | {:id ::d2 1240 | :fn ::done-counter 1241 | :in {:in p-id} 1242 | :opts {:done done-2 1243 | :counter (atom 0) 1244 | :n 2}} ; 2 + 2 from previous graph 1245 | {:id ::d3 1246 | :fn ::done-counter 1247 | :in {:in d-id} 1248 | :opts {:done done-3 1249 | :counter (atom 0) 1250 | :n 5}}]} 1251 | (q/graph) 1252 | (q/start-graph!))] 1253 | (is (done? done-2)) 1254 | (is (done? done-3)) 1255 | (is (= @logged 0)) 1256 | (is (= (q/all-graph-messages g) 1257 | {::s1 [{:x 1} {:x 2}] 1258 | ::q1 [{:x 2} {:x 3}] 1259 | p-id [{:q/type :q.type/snapshot 1260 | :q/proc-id p-id 1261 | :q/tailer-indices {p-tid 1}} 1262 | {:q/type :q.type/attempt-output 1263 | :q/message-index 1} 1264 | {:q/type :q.type/attempt-output 1265 | :q/message-index 1} 1266 | {:q/type :q.type/snapshot 1267 | :q/proc-id p-id 1268 | :q/tailer-indices {p-tid 2}} 1269 | {:q/type :q.type/attempt-output 1270 | :q/message-index 2} 1271 | {:q/type :q.type/snapshot 1272 | :q/proc-id p-id 1273 | :q/tailer-indices {p-tid 3}}] 1274 | d-id [{:q/type :q.type/snapshot 1275 | :q/proc-id d-id 1276 | :q/tailer-indices {d-tid 1}} 1277 | {:q/type :q.type/attempt-nil 1278 | :q/hash -1173148930} 1279 | {:q/type :q.type/snapshot 1280 | :q/proc-id d-id 1281 | :q/tailer-indices {d-tid 2}} 1282 | {:q/type :q.type/attempt-nil 1283 | :q/hash 1121832113} 1284 | {:q/type :q.type/snapshot 1285 | :q/proc-id d-id 1286 | :q/tailer-indices {d-tid 3}}]})) 1287 | (q/stop-graph! g) 1288 | (q/close-and-delete-graph! g true))))))) 1289 | 1290 | (def ^:private attempt-full 1291 | "Bind original here for use in test fn." 1292 | @#'q/attempt-full) 1293 | 1294 | (defn- attempt-and-write-then-interrupt 1295 | "Throws an unhandled interrupt immediately after both the attempt and 1296 | output write are completed." 1297 | [done] 1298 | (fn [appender msg] 1299 | (attempt-full appender msg) 1300 | (try 1301 | (throw (throw-interrupt!)) 1302 | (finally (deliver done true))))) 1303 | 1304 | (t/deftest exactly-once-attempt-and-write-then-interrupt-test 1305 | ;; The processor throws an unhandled interrupt after both the 1306 | ;; attempt and the output message are persisted. The processor then 1307 | ;; quits due to the unhandled interrupt. Without exactly once 1308 | ;; semantics, the message would be written again with at least once 1309 | ;; semantics on restart. However, using exactly once semantics the 1310 | ;; message is not persisted again after the graph is restarted. 1311 | (log/with-level :fatal 1312 | (let [p-id :cues.queue-test.graph.cues.queue-test/map-reduce 1313 | p-tid :cues.queue-test.graph.cues.queue-test.map-reduce.cues.queue-test/s1 1314 | d-id :cues.queue-test.graph.cues.queue-test/done 1315 | d-tid :cues.queue-test.graph.cues.queue-test.done.cues.queue-test/q1] 1316 | (let [done-1 (promise) 1317 | done-2 (promise)] 1318 | (with-redefs [q/attempt-full (attempt-and-write-then-interrupt done-1)] 1319 | (let [g (->> {:id ::graph 1320 | :processors [{:id ::s1} 1321 | {:id ::map-reduce 1322 | :in {:in ::s1} 1323 | :out {:out ::q1} 1324 | :opts {:map-fn inc 1325 | :to [:out]}} 1326 | {:id ::d2 1327 | :fn ::done-counter 1328 | :in {:in p-id} 1329 | :opts {:done done-2 1330 | :counter (atom 0) 1331 | :n 2}}]} 1332 | (q/graph) 1333 | (q/start-graph!))] 1334 | (q/send! g ::s1 {:x 1}) 1335 | (is (done? done-1)) 1336 | (is (= (q/all-graph-messages g) 1337 | {::s1 [{:x 1}] 1338 | ::q1 [{:x 2}] 1339 | p-id [{:q/type :q.type/snapshot 1340 | :q/proc-id p-id 1341 | :q/tailer-indices {p-tid 1}} 1342 | {:q/type :q.type/attempt-output 1343 | :q/message-index 1}]})) 1344 | (q/stop-graph! g) 1345 | (q/close-graph! g)))) 1346 | 1347 | ;; After restarting the graph the message is not re-delivered, 1348 | ;; and we can see there are not subsequent attempts in the try 1349 | ;; queue. 1350 | (let [done-2 (promise) 1351 | done-3 (promise)] 1352 | (let [g (->> {:id ::graph 1353 | :processors [{:id ::s1} 1354 | {:id ::map-reduce 1355 | :in {:in ::s1} 1356 | :out {:out ::q1} 1357 | :opts {:map-fn inc 1358 | :to [:out]}} 1359 | {:id ::done 1360 | :in {:in ::q1}} 1361 | {:id ::d2 1362 | :fn ::done-counter 1363 | :in {:in p-id} 1364 | :opts {:done done-2 1365 | :counter (atom 0) 1366 | :n 1}} ; 1 + 2 from previous graph 1367 | {:id ::d3 1368 | :fn ::done-counter 1369 | :in {:in d-id} 1370 | :opts {:done done-3 1371 | :counter (atom 0) 1372 | :n 3}}]} ; 3 + 0 from previous graph 1373 | (q/graph) 1374 | (q/start-graph!))] 1375 | (is (done? done-2)) 1376 | (is (done? done-3)) 1377 | (is (= (q/all-graph-messages g) 1378 | {::s1 [{:x 1}] 1379 | ::q1 [{:x 2}] 1380 | p-id [{:q/type :q.type/snapshot 1381 | :q/proc-id p-id 1382 | :q/tailer-indices {p-tid 1}} 1383 | {:q/type :q.type/attempt-output 1384 | :q/message-index 1} 1385 | {:q/type :q.type/snapshot 1386 | :q/proc-id p-id 1387 | :q/tailer-indices {p-tid 2}}] 1388 | d-id [{:q/type :q.type/snapshot 1389 | :q/proc-id d-id 1390 | :q/tailer-indices {d-tid 1}} 1391 | {:q/type :q.type/attempt-nil 1392 | :q/hash -1173148930} 1393 | {:q/type :q.type/snapshot 1394 | :q/proc-id d-id 1395 | :q/tailer-indices {d-tid 2}}]})) 1396 | (q/stop-graph! g) 1397 | (q/close-and-delete-graph! g true)))))) 1398 | 1399 | (t/deftest at-most-once-processor-unhandled-interrupt-test 1400 | ;; The processor :fn throws an unhandled interrupt during the method 1401 | ;; execution. The processor quits due to the unhandled 1402 | ;; interrupt. With at-most-once semantics, when the processor is 1403 | ;; restarted, it continues processing messages, having dropped the 1404 | ;; message that caused the error. 1405 | (log/with-level :fatal 1406 | (let [done (promise) 1407 | g (->> {:id ::graph 1408 | :strategy ::q/at-most-once 1409 | :processors [{:id ::s1} 1410 | {:id ::interrupt 1411 | :in {:in ::s1} 1412 | :out {:out ::q1} 1413 | :opts {:interrupt? true 1414 | :done done}} 1415 | {:id ::done 1416 | :in {:in ::q1}}]} 1417 | (q/graph) 1418 | (q/start-graph!))] 1419 | (q/send! g ::s1 {:x 1}) 1420 | (q/send! g ::s1 {:x 2}) 1421 | (is (done? done)) 1422 | (is (= (q/all-graph-messages g) 1423 | {::s1 [{:x 1} {:x 2}] 1424 | ::q1 []})) 1425 | (q/stop-graph! g) 1426 | (q/close-graph! g)) 1427 | 1428 | ;; After restarting the graph only the second message is delivered. 1429 | (let [done (promise) 1430 | g (->> {:id ::graph 1431 | :processors [{:id ::s1} 1432 | {:id ::interrupt 1433 | :in {:in ::s1} 1434 | :out {:out ::q1}} 1435 | {:id ::done 1436 | :in {:in ::q1} 1437 | :opts {:done done 1438 | :x-n 2}}]} 1439 | (q/graph) 1440 | (q/start-graph!))] 1441 | (is (done? done)) 1442 | (is (= (q/all-graph-messages g) 1443 | {::s1 [{:x 1} {:x 2}] 1444 | ::q1 [{:x 2}]})) 1445 | (q/stop-graph! g) 1446 | (q/close-and-delete-graph! g true)))) 1447 | 1448 | (t/deftest issue-1-test 1449 | (let [done-1 (promise) 1450 | done-2 (promise) 1451 | q (q/queue ::tmp {:queue-path ".cues-tmp"}) 1452 | t1 (q/tailer q) 1453 | t2 (q/tailer q) 1454 | a (q/appender q)] 1455 | (try 1456 | (future (deliver done-1 (q/read!! t1))) 1457 | (future (deliver done-2 (q/read!! t2))) 1458 | (Thread/sleep 1) 1459 | (q/write a {:x 1}) 1460 | (is (= (done? done-1) {:x 1})) 1461 | (is (= (done? done-2) {:x 1})) 1462 | (finally 1463 | (q/delete-queue! q true) 1464 | (cutil/delete-file (io/file ".cues-tmp")))))) 1465 | 1466 | (def stress-fixtures 1467 | (t/join-fixtures [qt/with-warn])) 1468 | 1469 | (defn stress-test 1470 | [n] 1471 | (stress-fixtures 1472 | (fn [] 1473 | (let [q1-done (promise) 1474 | q2-done (promise) 1475 | timeout (max (/ n 10) 1000)] 1476 | (qt/with-graph-and-delete 1477 | [g {:id ::graph 1478 | :queue-opts {::q/default {:queue-meta false}} 1479 | :processors [{:id ::s1} 1480 | {:id ::s2} 1481 | {:id ::map-reduce 1482 | :in {:in-1 ::s1 1483 | :in-2 ::s2} 1484 | :out {:out-1 ::q1 1485 | :out-2 ::q2} 1486 | :opts {:reduce-fn + 1487 | :to [:out-1 :out-2]}} 1488 | {:id ::d1 1489 | :fn ::done-counter 1490 | :in {:in ::q1} 1491 | :opts {:done q1-done 1492 | :counter (atom 0) 1493 | :n n}} 1494 | {:id ::d2 1495 | :fn ::done-counter 1496 | :in {:in ::q2} 1497 | :opts {:done q2-done 1498 | :counter (atom 0) 1499 | :n n}}]}] 1500 | {:success? (time 1501 | (do 1502 | (time 1503 | (dotimes [n n] 1504 | (q/send! g ::s1 {:x n}) 1505 | (q/send! g ::s2 {:x n}))) 1506 | (and (done? q1-done timeout) 1507 | (done? q2-done timeout)))) 1508 | :counts (->> g 1509 | (:queues) 1510 | (cutil/map-vals (comp count q/all-messages)))}))))) 1511 | 1512 | (comment 1513 | ;; This measures a single round-trip read + write, with 1514 | ;; blocking. The messages being serialized on these queues are 1515 | ;; generally between 114-117 Bytes in size. 1516 | 1517 | (b/quick-bench 1518 | (q/write a {:x 1})) 1519 | 1520 | "Evaluation count : 865068 in 6 samples of 144178 calls. 1521 | Execution time mean : 690.932746 ns 1522 | Execution time std-deviation : 7.290161 ns 1523 | Execution time lower quantile : 683.505105 ns ( 2.5%) 1524 | Execution time upper quantile : 698.417843 ns (97.5%) 1525 | Overhead used : 2.041010 ns" 1526 | 1527 | (b/quick-bench 1528 | (do (q/write a {:x 1}) 1529 | (q/read!! t))) 1530 | "Evaluation count : 252834 in 6 samples of 42139 calls. 1531 | Execution time mean : 2.389696 µs 1532 | Execution time std-deviation : 64.305722 ns 1533 | Execution time lower quantile : 2.340823 µs ( 2.5%) 1534 | Execution time upper quantile : 2.466880 µs (97.5%) 1535 | Overhead used : 2.035620 ns" 1536 | 1537 | (qt/stress-test 1000000) 1538 | "Elapsed time: 4312.960375 msecs" 1539 | "Elapsed time: 29921.640709 msecs" 1540 | {:success? true 1541 | :counts {:cues.test/error 0 1542 | :cues.queue-test/s1 1000000 1543 | :cues.queue-test/s2 1000000 1544 | :cues.queue-test/q1 1000000 1545 | :cues.queue-test/q2 1000000}}) 1546 | --------------------------------------------------------------------------------