├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── pom.xml └── src ├── main └── java │ └── no │ └── systek │ └── dataflow │ ├── ContextSwitcher.java │ ├── PriorityTaskQueue.java │ ├── Step.java │ ├── StepExecutor.java │ ├── Steps.java │ └── steps │ ├── CollectorStep.java │ ├── ConditionalStep.java │ ├── ListStep.java │ ├── PairJoinStep.java │ └── SourceStep.java └── test └── java └── no └── systek └── dataflow ├── AbstractStepTest.java ├── CappuccinoTest.java ├── PriorityTaskQueueTest.java ├── StepTest.java └── types ├── BlackCoffee.java ├── Cappuccino.java ├── FoamedMilk.java ├── GrindedCoffee.java ├── Order.java └── Water.java /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | target/ 3 | *.iml 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | script: mvn verify -Pcoverage -B 3 | jdk: 4 | - oraclejdk8 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/systek/dataflow.svg?branch=master)](https://travis-ci.org/systek/dataflow) 2 | [![Coverage Status](https://coveralls.io/repos/github/systek/dataflow/badge.svg)](https://coveralls.io/github/systek/dataflow) 3 | 4 | # dataflow 5 | A Java library, which lets you define dependencies between tasks (called steps) and then execute the entire graph. 6 | Steps which have no unresolved dependencies are executed first - in parallel. 7 | 8 | ## Usage 9 | ``` 10 | 11 | dataflow 12 | no.systek.dataflow 13 | 0.2 14 | 15 | ``` 16 | 17 | ## Requirements 18 | - Java8 19 | 20 | There are no dependencies to other libraries except for SLF4j. 21 | 22 | ## "Step" 23 | Similar to "[actors](https://en.wikipedia.org/wiki/Actor_model)", a step is a piece of work (/task) which is executed as 24 | some input arrives and can produce one or more outputs during execution. 25 | 26 | Steps are then linked together to form a graph by defining dependencies between them. Each time a step produces an output, 27 | this output is then automatically distributed to it's child steps which get ready to be executed. 28 | 29 | ### Simple Example 30 | From the [CappuccinoTest](https://github.com/systek/dataflow/blob/master/src/test/java/no/systek/dataflow/CappuccinoTest.java#L29), 31 | the following steps produce one cappuccino: 32 | 33 | ``` 34 | GrindBeans-------------+ 35 | v 36 | HeatWater-----------> brew ----+ 37 | | 38 | v 39 | FoamMilk-----------------> Cappuccino 40 | ``` 41 | This graph is defined in code like this: 42 | 43 | ```java 44 | cappuccino.dependsOn(brew.output()); 45 | cappuccino.dependsOn(foamMilk.output()); 46 | 47 | brew.dependsOn(heatWater.output()); 48 | brew.dependsOn(grindBeans.output()); 49 | ``` 50 | To fullfill the cappuccino step it needs input from both the brew and the foamMilk step. Only after both inputs 51 | are available, execution of the cappuccino step is scheduled. 52 | And the brew step cannot start before it has received its required inputs from heatWater and GrindBeans. And so on. 53 | 54 | ## More complex graphs 55 | A [Step](https://github.com/systek/dataflow/blob/master/src/main/java/no/systek/dataflow/Step.java) can be easily 56 | extended to achieve rich capabilities like *collectors*, *conditional routing* and even *loops*!. 57 | 58 | ### Conditional step 59 | The library includes a [ConditionalStep](https://github.com/systek/dataflow/blob/master/src/main/java/no/systek/dataflow/steps/ConditionalStep.java) which allows for 2-way conditional routing. For example: 60 | 61 | ``` 62 | PickAppleFromTree 63 | | 64 | v 65 | InGoodCondition? 66 | | 67 | +--no--+--yes---+ 68 | | | 69 | v v 70 | ThrowAway Collect 71 | ``` 72 | 73 | ### Collector step 74 | After a fork-out where processing is done in parallel, it might be desirable to join the output of those parallel steps again before continuing. 75 | This can be done with so called [CollectorStep](https://github.com/systek/dataflow/blob/master/src/main/java/no/systek/dataflow/steps/CollectorStep.java). 76 | 77 | For example, image you want to process an order which contains multiple order lines. Each order line is processed in parallel but shipping and invoicing is only once done once for entire order: 78 | 79 | ``` 80 | orderLineSplitter 81 | | 82 | v 83 | parallelOrderLineProcessor // started in parallel for each line 84 | | 85 | v 86 | lineNeedsShipping 87 | | | 88 |   +--yes----+ +---no-----+ 89 | | | 90 | v | 91 | collectForShipping v 92 | | collectNoShipping 93 | v | 94 | shipAllItems | 95 | | | 96 | +------> createInvoice <---+ 97 | | 98 | v 99 | sendInvoice 100 | | 101 | v 102 | finished 103 | ``` 104 | 105 | ```java 106 | // define the steps 107 | Step collectForShipping = Steps.newCollector(Integer.MAX_VALUE) 108 | Step collectNoShipping = Steps.newCollector(Integer.MAX_VALUE) 109 | // ... 110 | 111 | # setup their dependencies 112 | finished.dependsOn(sendInvoice.output()) 113 | 114 | sendInvoice.dependsOn(createInvoice.output()) 115 | 116 | createInvoice.dependsOn(shipAllItems.output()) 117 | createInvoice.dependsOn(collectNoShipping.output()) 118 | 119 | collectNoShipping.dependsOn(lineNeedsShipping.ifFalse()) 120 | 121 | shipAllItems.dependsOn(collectForShipping.output()) 122 | collectForShipping.dependsOn(lineNeedsShipping.ifTrue()) 123 | 124 | lineNeedsShipping.dependsOn(parallelOrderLineProcessor.output()) 125 | 126 | parallelOrderLineProcessor.dependsOn(orderLineSplitter.output()) 127 | 128 | // and now execute the entire graph with order as input 129 | finished.executeTasksAndAwaitDone(order); 130 | ``` 131 | 132 | In this example, the step parallelOrderLineProcessor is permitted to execute in parallel. The orderLineSplitter 133 | takes the entire order as input and procudes outputs for each order line. As soon as a new order line is 134 | sendt to parallelOrderLineProcessor, processing starts in parallel. After order line processing, the results are 135 | collected depending on whether shipping is needed or not. 136 | 137 | #### Finish collecting? 138 | But how does a collector step know when to proceed, e.g. that there will be no more inputs arriving? 139 | This information is derived from the fact that there are no more steps executing and thus all 140 | are awaiting more input. A collector step schedules a cleanup task in the taskScheduler with a lower pririty, 141 | which gets only executed once all other steps have finished. See [CollectorStep](https://github.com/systek/dataflow/blob/master/src/main/java/no/systek/dataflow/steps/CollectorStep.java#L38). 142 | 143 | 144 | ### Loops 145 | It is also possible to use conditions and loops to build more complex graphs. This is really useful when you have 146 | a more complex business transaction which require *optimistic locking*. 147 | 148 | For example if the heated water is not hot enough, it is re-heated again. 149 | 150 | ``` 151 | GrindBeans---------------------------+ 152 | v 153 | HeatWater----> HotEnough?---yes---> brew ------+ 154 | ^ | | 155 | +---------no---+ | 156 | v 157 | FoamMilk--------------------------------> Cappuccino 158 | ``` 159 | 160 | Expressed in code like this: 161 | 162 | ```java 163 | cappuccino.dependsOn(brew.output()); 164 | cappuccino.dependsOn(foamMilk.output()); 165 | 166 | brew.dependsOn(waterHotEnough.ifTrue()); 167 | brew.dependsOn(grindBeans.output()); 168 | 169 | waterHotEnough.dependsOn(heatWater.output()); 170 | heatWater.dependsOn(waterHotEnough.ifFalse()); 171 | ``` 172 | 173 | ## Parallel execution 174 | Like in the actor-model, a step has a *mailbox* in which inbound input values are queued. As soon 175 | as a new input value is queued in this mailbox, the step gets ready to be executed. 176 | 177 | All steps which are ready to be executed, thus not awaiting some input, are executed in parallel. 178 | The max number of concurrent executions is configurable via the included task scheduler 179 | [PriorityTaskQueue](https://github.com/systek/dataflow/blob/master/src/main/java/no/systek/dataflow/PriorityTaskQueue.java). 180 | In our cappucino example above, GrindBeans, HeatWater and FoamMilk will start executing in parallel 181 | because those do not depend on each other. 182 | 183 | Unlike actors, where a single actor can never be executed in parallel, a "step" in this library ***can*** be 184 | executed in parallel as soon as more input values become available while it is already being 185 | executed. This is configurable by setting the step property "maxParallelExecution" to larger 186 | than 1. Of course any internal state in the step becomes now subject to concurrent access and must 187 | be protected accordingly. 188 | 189 | This is really usefull if you have a stateless step which is expected to process many inputs. 190 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 4.0.0 6 | 7 | dataflow 8 | no.systek.dataflow 9 | 0.3-SNAPSHOT 10 | jar 11 | 12 | dataflow 13 | Dataflow execution library for java 14 | https://github.com/systek/dataflow 15 | 16 | 17 | 18 | Sebastian Dehne 19 | sebastian.dehne@systek.no 20 | 21 | 22 | 23 | 24 | 25 | ossrh 26 | https://oss.sonatype.org/content/repositories/snapshots 27 | 28 | 29 | ossrh 30 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 31 | 32 | 33 | 34 | 35 | 36 | Apache License, Version 2.0 37 | http://www.apache.org/licenses/LICENSE-2.0.txt 38 | repo 39 | 40 | 41 | 42 | 43 | https://github.com/systek/dataflow 44 | 45 | 46 | 47 | 1.8 48 | UTF-8 49 | UTF-8 50 | 51 | 52 | 53 | 54 | 55 | release 56 | 57 | 58 | 59 | org.apache.maven.plugins 60 | maven-gpg-plugin 61 | 1.6 62 | 63 | 64 | sign-artifacts 65 | verify 66 | 67 | sign 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | coverage 77 | 78 | 79 | 80 | org.eluder.coveralls 81 | coveralls-maven-plugin 82 | 4.3.0 83 | 84 | 85 | default-report 86 | verify 87 | 88 | report 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | maven-compiler-plugin 102 | 3.5.1 103 | 104 | 1.8 105 | 1.8 106 | 107 | 108 | 109 | org.apache.maven.plugins 110 | maven-source-plugin 111 | 112 | 113 | attach-sources 114 | 115 | jar-no-fork 116 | 117 | 118 | 119 | 120 | 121 | org.apache.maven.plugins 122 | maven-javadoc-plugin 123 | 124 | 125 | attach-javadocs 126 | 127 | jar 128 | 129 | 130 | 131 | 132 | 133 | org.apache.maven.plugins 134 | maven-surefire-plugin 135 | 2.19.1 136 | 137 | ${argLine} 138 | 139 | 140 | 141 | org.sonatype.plugins 142 | nexus-staging-maven-plugin 143 | 1.6.7 144 | true 145 | 146 | ossrh 147 | https://oss.sonatype.org/ 148 | true 149 | 150 | 151 | 152 | org.jacoco 153 | jacoco-maven-plugin 154 | 0.7.8 155 | 156 | 157 | default-prepare-agent 158 | 159 | prepare-agent 160 | 161 | 162 | 163 | default-report 164 | prepare-package 165 | 166 | report 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | org.slf4j 177 | slf4j-api 178 | 1.7.21 179 | 180 | 181 | junit 182 | junit 183 | 4.12 184 | test 185 | 186 | 187 | org.hamcrest 188 | hamcrest-junit 189 | 2.0.0.0 190 | test 191 | 192 | 193 | 194 | 195 | -------------------------------------------------------------------------------- /src/main/java/no/systek/dataflow/ContextSwitcher.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.slf4j.MDC; 6 | 7 | import java.util.Map; 8 | import java.util.Optional; 9 | import java.util.function.Consumer; 10 | import java.util.function.Supplier; 11 | 12 | public class ContextSwitcher { 13 | private static final Logger LOGGER = LoggerFactory.getLogger(ContextSwitcher.class); 14 | 15 | /** 16 | * Simple wrapper which copies over the context (MDC and correlation) to the executing thread and 17 | * logs uncaught exceptions 18 | */ 19 | public static Runnable wrap(Runnable in, Supplier correlationIdProvider, Consumer correlationIdSetter) { 20 | final Optional> context = Optional.ofNullable(MDC.getCopyOfContextMap()); 21 | final Optional korrelasjonsId = Optional.ofNullable(correlationIdProvider.get()); 22 | return () -> { 23 | Optional> contextBackup = Optional.ofNullable(MDC.getCopyOfContextMap()); 24 | final Optional backupKorrelasjonsId = Optional.ofNullable(correlationIdProvider.get()); 25 | context.ifPresent(MDC::setContextMap); 26 | korrelasjonsId.ifPresent(correlationIdSetter); 27 | try { 28 | in.run(); 29 | } catch (Exception e) { 30 | LOGGER.error(e.getMessage(), e); 31 | throw e; 32 | } finally { 33 | MDC.clear(); 34 | contextBackup.ifPresent(MDC::setContextMap); 35 | backupKorrelasjonsId.ifPresent(correlationIdSetter); 36 | } 37 | }; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/no/systek/dataflow/PriorityTaskQueue.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow; 2 | 3 | import java.util.LinkedList; 4 | import java.util.List; 5 | import java.util.Queue; 6 | import java.util.concurrent.Callable; 7 | import java.util.concurrent.ExecutorService; 8 | import java.util.concurrent.LinkedBlockingQueue; 9 | import java.util.concurrent.TimeUnit; 10 | import java.util.concurrent.atomic.AtomicInteger; 11 | import java.util.concurrent.locks.Condition; 12 | import java.util.concurrent.locks.Lock; 13 | import java.util.concurrent.locks.ReentrantLock; 14 | import java.util.function.Consumer; 15 | import java.util.function.Supplier; 16 | 17 | /** 18 | * Schedules tasks ordered by priority 19 | */ 20 | public class PriorityTaskQueue { 21 | public static final int HIGHEST_PRIORITY = 1; 22 | 23 | private final Lock lock; 24 | private final Condition taskCompleted; 25 | private final int maxParallelTasks; 26 | private final Supplier correlationIdProvider; 27 | private final Consumer correlationIdSetter; 28 | 29 | // guarded by "lock" 30 | private final AtomicInteger runningTasks = new AtomicInteger(0); 31 | private final List>> queues; 32 | 33 | public PriorityTaskQueue(int maxParallelTasks, 34 | Supplier correlationIdProvider, 35 | Consumer correlationIdSetter) { 36 | 37 | this.lock = new ReentrantLock(); 38 | this.taskCompleted = this.lock.newCondition(); 39 | this.maxParallelTasks = maxParallelTasks; 40 | this.queues = new LinkedList<>(); 41 | this.correlationIdProvider = correlationIdProvider; 42 | this.correlationIdSetter = correlationIdSetter; 43 | } 44 | 45 | public void addTask(int priority, Consumer task) { 46 | if (priority < HIGHEST_PRIORITY) { 47 | throw new RuntimeException("Priority cannot be lower than 1"); 48 | } 49 | locked(() -> { 50 | while (queues.size() < priority) { 51 | queues.add(new LinkedBlockingQueue<>()); 52 | } 53 | queues.get(priority - 1).offer(task); 54 | }); 55 | } 56 | 57 | /** 58 | * Executes all queued tasks until all done. It tries to complete all tasks at the highest 59 | * priority first before moving to the next priority. If a new task got scheduled at a higher 60 | * priority in the mean time, it moves back to the higher priority 61 | * 62 | * @return true if all queues has been done, false if timeout was reached 63 | */ 64 | public boolean executeTasksAndAwaitDone( 65 | ExecutorService executorService, 66 | Consumer exceptionListener, 67 | long timeout, 68 | TimeUnit unit) { 69 | 70 | final long deadLine = System.currentTimeMillis() + unit.toMillis(timeout); 71 | 72 | return locked(() -> { 73 | 74 | int currentPriority = 0; 75 | boolean foundTasksAtPriority = false; 76 | while (true) { 77 | 78 | if (System.currentTimeMillis() > deadLine) { 79 | return false; 80 | } 81 | if (currentPriority >= queues.size()) { 82 | return true; 83 | } 84 | 85 | Queue> tasksAtCurrentPriority = queues.get(currentPriority); 86 | 87 | if (!tasksAtCurrentPriority.isEmpty()) { 88 | foundTasksAtPriority = true; 89 | if (tryScheduleTask(executorService, tasksAtCurrentPriority.peek(), exceptionListener)) { 90 | tasksAtCurrentPriority.poll(); 91 | continue; 92 | } 93 | } 94 | 95 | if (runningTasks.get() > 0) { 96 | try { 97 | taskCompleted.await(Math.max(1, deadLine - System.currentTimeMillis()), TimeUnit.MILLISECONDS); 98 | } catch (InterruptedException ignored) { 99 | } 100 | continue; 101 | } 102 | 103 | currentPriority = !foundTasksAtPriority ? currentPriority + 1 : 0; 104 | foundTasksAtPriority = false; 105 | } 106 | }); 107 | 108 | } 109 | 110 | /** 111 | * Submit a task to the thread pool 112 | * 113 | * @return true is the task was submitted, or false if max parallel tasks has been reached 114 | */ 115 | private boolean tryScheduleTask( 116 | ExecutorService executorService, 117 | Consumer task, 118 | Consumer exceptionListener) { 119 | 120 | if (runningTasks.incrementAndGet() <= maxParallelTasks) { 121 | executorService.submit(ContextSwitcher.wrap(() -> { 122 | try { 123 | task.accept(this); 124 | } catch (Exception e) { 125 | exceptionListener.accept(e); 126 | } finally { 127 | locked(() -> { 128 | runningTasks.decrementAndGet(); 129 | taskCompleted.signalAll(); 130 | }); 131 | } 132 | }, correlationIdProvider, correlationIdSetter)); 133 | return true; 134 | } else { 135 | // give it back 136 | runningTasks.decrementAndGet(); 137 | return false; 138 | } 139 | } 140 | 141 | private void locked(Runnable r) { 142 | locked(() -> { 143 | r.run(); 144 | return false; 145 | }); 146 | } 147 | 148 | private T locked(Callable r) { 149 | lock.lock(); 150 | try { 151 | try { 152 | return r.call(); 153 | } catch (Exception e) { 154 | throw new RuntimeException(e); 155 | } 156 | } finally { 157 | lock.unlock(); 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/main/java/no/systek/dataflow/Step.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow; 2 | 3 | import java.util.HashSet; 4 | import java.util.LinkedList; 5 | import java.util.List; 6 | import java.util.Queue; 7 | import java.util.concurrent.ConcurrentLinkedQueue; 8 | import java.util.concurrent.ExecutorService; 9 | import java.util.concurrent.TimeUnit; 10 | import java.util.concurrent.atomic.AtomicInteger; 11 | import java.util.function.Consumer; 12 | 13 | /** 14 | * Similar to "actors", a step is a piece of work which is executed as some input arrives and can produce 15 | * one or more outputs. 16 | *

17 | * The config parameter "maxParallelExecution" controls how many times this Step can be started in parallel 18 | * to handle inbound events concurrently. Set it to 1 to disallow concurrent execution, in which case internal state 19 | * is protected from concurrent access. 20 | *

21 | * Steps can be chained together to (complex) graphs by defining dependencies between then, including loop scenarios. 22 | *

23 | * See CappuccinoTest 24 | *

25 | */ 26 | @SuppressWarnings("WeakerAccess") 27 | public abstract class Step { 28 | 29 | private final String name; 30 | private final int maxParallelExecution; 31 | private final List> parents = new LinkedList<>(); 32 | private final List> children = new LinkedList<>(); 33 | private final Queue msgBox = new ConcurrentLinkedQueue<>(); 34 | private final AtomicInteger scheduledJobs = new AtomicInteger(); 35 | private final AtomicInteger lock = new AtomicInteger(); 36 | private volatile int graphDepth; 37 | protected volatile Consumer onResult; 38 | 39 | public Step(int maxParallelExecution) { 40 | this(null, maxParallelExecution); 41 | } 42 | 43 | public Step(String name, int maxParallelExecution) { 44 | this.name = name == null ? this.getClass().getSimpleName() : name; 45 | this.maxParallelExecution = maxParallelExecution; 46 | } 47 | 48 | public String getName() { 49 | return name; 50 | } 51 | 52 | public boolean executeTasksAndAwaitDone( 53 | PriorityTaskQueue taskQueue, 54 | ExecutorService executorService, 55 | Consumer exceptionListener, 56 | Object input, 57 | Consumer onResult, 58 | long timeout, 59 | TimeUnit unit) { 60 | 61 | if (!children.isEmpty()) { 62 | throw new RuntimeException("This step has children; please start executing at the tail of the graph"); 63 | } 64 | this.onResult = onResult; 65 | 66 | // walk the step graph: configure the graph depth on each step and find the root steps 67 | HashSet> roots = new HashSet<>(); 68 | configureTreeAndFindRoots(new HashSet<>(), roots); 69 | 70 | // start execution by scheduling tasks for all roots 71 | roots.forEach(rootStep -> rootStep.post(input, taskQueue)); 72 | 73 | return taskQueue.executeTasksAndAwaitDone(executorService, exceptionListener, timeout, unit); 74 | } 75 | 76 | public void dependsOn(DependencyCreator dependency) { 77 | addParent(dependency.step); 78 | dependency.link(this); 79 | } 80 | 81 | public DependencyCreator output() { 82 | return new DependencyCreator<>((Step) this); 83 | } 84 | 85 | public void post(I input, PriorityTaskQueue taskQueue) { 86 | if (input != null) { 87 | msgBox.offer(input); 88 | } 89 | 90 | // try to schedule a new task on the thread pool which handles this new input 91 | tryScheduleNextJob(taskQueue); 92 | } 93 | 94 | protected abstract void run(I input, Consumer onResult); 95 | 96 | protected void afterRun(PriorityTaskQueue taskQueue) { 97 | } 98 | 99 | protected int configureTreeAndFindRoots(HashSet> visited, HashSet> roots) { 100 | try { 101 | if (!visited.add(this)) { 102 | return this.graphDepth; 103 | } 104 | 105 | int myDepth = 1; 106 | 107 | if (parents.isEmpty() || (parents.size() == 1 && visited.contains(parents.get(0)))) { 108 | roots.add((Step) this); 109 | } else { 110 | for (Step parent : parents) { 111 | int parentLevel = parent.configureTreeAndFindRoots(visited, roots); 112 | if (parentLevel > myDepth) { 113 | myDepth = parentLevel; 114 | } 115 | } 116 | myDepth++; 117 | } 118 | 119 | this.graphDepth = myDepth; 120 | 121 | return myDepth; 122 | } finally { 123 | visited.remove(this); 124 | } 125 | } 126 | 127 | private void tryScheduleNextJob(PriorityTaskQueue taskQueue) { 128 | // only one thread at a time here 129 | if (lock.getAndIncrement() == 0) { 130 | try { 131 | while (msgBox.peek() != null && scheduledJobs.get() < maxParallelExecution) { 132 | scheduledJobs.incrementAndGet(); 133 | I input = msgBox.poll(); 134 | taskQueue.addTask(PriorityTaskQueue.HIGHEST_PRIORITY, pq -> { 135 | try { 136 | run(input, output -> onOutputAvailable(output, pq)); 137 | afterRun(pq); 138 | } finally { 139 | scheduledJobs.decrementAndGet(); 140 | tryScheduleNextJob(pq); 141 | } 142 | }); 143 | } 144 | } finally { 145 | if (lock.getAndSet(0) != 1) { 146 | // another thread tried to enter this block while we had the lock, re-run it in case new messages 147 | // have arrived 148 | taskQueue.addTask(PriorityTaskQueue.HIGHEST_PRIORITY, this::tryScheduleNextJob); 149 | } 150 | } 151 | } 152 | } 153 | 154 | protected void onOutputAvailable(O output, PriorityTaskQueue pq) { 155 | if (children.isEmpty()) { 156 | onResult.accept(output); 157 | } else { 158 | children.forEach(s -> s.post(output, pq)); 159 | } 160 | } 161 | 162 | protected void addParent(Step parent) { 163 | this.parents.add(parent); 164 | } 165 | 166 | protected List> getChildren() { 167 | return children; 168 | } 169 | 170 | protected Integer getGraphDepth() { 171 | return graphDepth; 172 | } 173 | 174 | protected static class DependencyCreator { 175 | public final Step step; 176 | 177 | public DependencyCreator(Step step) { 178 | this.step = step; 179 | } 180 | 181 | public void link(Step child) { 182 | step.children.add(child); 183 | } 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/main/java/no/systek/dataflow/StepExecutor.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.util.LinkedList; 7 | import java.util.List; 8 | import java.util.concurrent.CopyOnWriteArrayList; 9 | import java.util.concurrent.ExecutorService; 10 | import java.util.concurrent.TimeUnit; 11 | import java.util.function.Consumer; 12 | import java.util.function.Supplier; 13 | 14 | /** 15 | * Helper to execute a graph of steps with default error handling 16 | */ 17 | @SuppressWarnings({ "WeakerAccess", "unused" }) 18 | public class StepExecutor { 19 | private final static Logger LOGGER = LoggerFactory.getLogger(StepExecutor.class); 20 | 21 | private final ExecutorService executorService; 22 | private final Consumer correlationIdSettter; 23 | private final Supplier correlationIdGetter; 24 | private final int maxParallelTasks; 25 | private final long timeout; 26 | private final TimeUnit timeUnit; 27 | 28 | public StepExecutor( 29 | ExecutorService executorService, 30 | Consumer correlationIdSettter, 31 | Supplier correlationIdGetter, 32 | int maxParallelTasks, 33 | long timeout, 34 | TimeUnit timeUnit) { 35 | 36 | this.executorService = executorService; 37 | this.correlationIdSettter = correlationIdSettter; 38 | this.correlationIdGetter = correlationIdGetter; 39 | this.maxParallelTasks = maxParallelTasks; 40 | this.timeout = timeout; 41 | this.timeUnit = timeUnit; 42 | } 43 | 44 | public List executeList(Step tail, Object input) { 45 | 46 | List exceptions = new CopyOnWriteArrayList<>(); 47 | List results = new CopyOnWriteArrayList<>(); 48 | 49 | if (!tail.executeTasksAndAwaitDone( 50 | new PriorityTaskQueue(maxParallelTasks, correlationIdGetter, correlationIdSettter), 51 | executorService, 52 | exceptions::add, 53 | input, 54 | results::add, 55 | timeout, 56 | timeUnit)) { 57 | throw new RuntimeException("Timeout during execution"); 58 | } 59 | 60 | if (!exceptions.isEmpty()) { 61 | exceptions.forEach(e -> LOGGER.error("", e)); 62 | throw new RuntimeException("One or more exceptions caught during execution, see logging"); 63 | } 64 | 65 | return new LinkedList<>(results); 66 | } 67 | 68 | public List executeList(Step tail) { 69 | return executeList(tail, new Object()); 70 | } 71 | 72 | public O execute(Step tail, Object input) { 73 | List results = executeList(tail, input); 74 | return results.isEmpty() ? null : results.get(0); 75 | } 76 | 77 | public O execute(Step tail) { 78 | return execute(tail, new Object()); 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/no/systek/dataflow/Steps.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow; 2 | 3 | import no.systek.dataflow.steps.*; 4 | 5 | import java.util.List; 6 | import java.util.function.*; 7 | 8 | @SuppressWarnings({"WeakerAccess", "SameParameterValue", "unused"}) 9 | public final class Steps { 10 | 11 | public static SourceStep newSource(Supplier supplier) { 12 | return new SourceStep(null, 1) { 13 | @Override 14 | protected O get() { 15 | return supplier.get(); 16 | } 17 | }; 18 | } 19 | 20 | public static Step newSingle(Function func) { 21 | return new SingleStep(null) { 22 | @Override 23 | O execute(I input) { 24 | return func.apply(input); 25 | } 26 | }; 27 | } 28 | 29 | public static Step newParallel(Function func) { 30 | return new ParallelStep(null) { 31 | @Override 32 | O execute(I input) { 33 | return func.apply(input); 34 | } 35 | }; 36 | } 37 | 38 | public static CollectorStep newCollector(int bufferSize) { 39 | return new CollectorStep<>(null, bufferSize); 40 | } 41 | 42 | public static PairJoinStep newJoiner( 43 | Predicate isLeft, 44 | BiFunction func) { 45 | 46 | return new PairJoinStep(null) { 47 | @Override 48 | protected O join(Ileft left, Iright right) { 49 | return func.apply(left, right); 50 | } 51 | 52 | @Override 53 | protected boolean isLeft(Object input) { 54 | return isLeft.test(input); 55 | } 56 | }; 57 | } 58 | 59 | public static SimpleConditionalStep newCondition(Predicate test) { 60 | return new SimpleConditionalStep(null) { 61 | @Override 62 | boolean test(T input) { 63 | return test.test(input); 64 | } 65 | }; 66 | } 67 | 68 | public static ListStep newParallelListStep(Function, List> func) { 69 | return new ListStep(null) { 70 | @Override 71 | protected List execute(List in) { 72 | return func.apply(in); 73 | } 74 | }; 75 | } 76 | 77 | public static abstract class SimpleConditionalStep extends ConditionalStep { 78 | 79 | public SimpleConditionalStep(String name) { 80 | super(name, Integer.MAX_VALUE); 81 | } 82 | 83 | @Override 84 | protected void run(T input, BiConsumer onResult) { 85 | onResult.accept(test(input), input); 86 | } 87 | 88 | abstract boolean test(T input); 89 | 90 | } 91 | 92 | public static abstract class ParallelStep extends SimpleStep { 93 | public ParallelStep(String name) { 94 | super(name, Integer.MAX_VALUE); 95 | } 96 | } 97 | 98 | public static abstract class SingleStep extends SimpleStep { 99 | public SingleStep(String name) { 100 | super(name, 1); 101 | } 102 | } 103 | 104 | public static abstract class SimpleStep extends Step { 105 | 106 | public SimpleStep(String name, int maxParallelExecution) { 107 | super(name, maxParallelExecution); 108 | } 109 | 110 | @Override 111 | protected void run(I input, Consumer onResult) { 112 | onResult.accept(execute(input)); 113 | } 114 | 115 | abstract O execute(I input); 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /src/main/java/no/systek/dataflow/steps/CollectorStep.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow.steps; 2 | 3 | import java.util.LinkedList; 4 | import java.util.List; 5 | import java.util.function.Consumer; 6 | 7 | import no.systek.dataflow.PriorityTaskQueue; 8 | import no.systek.dataflow.Step; 9 | 10 | public class CollectorStep extends Step> { 11 | private final T CLEANUP = (T) "CLEANUP"; 12 | 13 | private final List items; 14 | private final int bufferSize; 15 | private boolean scheduledCleanup; 16 | 17 | public CollectorStep(String name, int bufferSize) { 18 | super(name, 1); 19 | this.bufferSize = bufferSize; 20 | this.items = new LinkedList<>(); 21 | } 22 | 23 | @Override 24 | protected void run(T input, Consumer> onResult) { 25 | if (CLEANUP.equals(input)) { 26 | scheduledCleanup = false; 27 | pushItems(onResult); 28 | } else { 29 | if (items.size() >= bufferSize) { 30 | pushItems(onResult); 31 | } 32 | items.add(input); 33 | } 34 | } 35 | 36 | @Override 37 | protected void afterRun(PriorityTaskQueue priorityTaskQueue) { 38 | // schedule a cleanup tasks at a lower priority such that if there are no more 39 | // steps executing (thus waiting for some input), then this step should run using 40 | // the items it got so far. 41 | 42 | // do this only once, the the first item is scheduled 43 | if (!items.isEmpty() && !scheduledCleanup) { 44 | scheduledCleanup = true; 45 | priorityTaskQueue.addTask(getGraphDepth(), pq2 -> post(CLEANUP, pq2)); 46 | } 47 | } 48 | 49 | private void pushItems(Consumer> onResult) { 50 | List items = new LinkedList<>(this.items); 51 | this.items.clear(); 52 | onResult.accept(items); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/no/systek/dataflow/steps/ConditionalStep.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow.steps; 2 | 3 | import java.util.LinkedList; 4 | import java.util.List; 5 | import java.util.function.BiConsumer; 6 | import java.util.function.Consumer; 7 | 8 | import no.systek.dataflow.PriorityTaskQueue; 9 | import no.systek.dataflow.Step; 10 | 11 | /** 12 | * Conditional step for 2-way step dependencies which can route dynamically depending on the conditional 13 | * output 14 | *

15 | * See CappuccinoTest 16 | */ 17 | @SuppressWarnings("WeakerAccess") 18 | public abstract class ConditionalStep extends Step { 19 | 20 | private final List> falseChildren = new LinkedList<>(); 21 | 22 | public ConditionalStep(String name, int maxParallelExecution) { 23 | super(name, maxParallelExecution); 24 | } 25 | 26 | protected abstract void run(I input, BiConsumer onResult); 27 | 28 | @Override 29 | protected final void run(I input, Consumer onResult) { 30 | run(input, (condition, result) -> onResult.accept((O) new ConditionalResult<>(condition, result))); 31 | } 32 | 33 | @Override 34 | protected final void onOutputAvailable(O output, PriorityTaskQueue pq) { 35 | ConditionalResult r = (ConditionalResult) output; 36 | List> next = r.condition ? getChildren() : falseChildren; 37 | if (next.isEmpty()) { 38 | onResult.accept(r.output); 39 | } else { 40 | next.forEach(s -> s.post(r.output, pq)); 41 | } 42 | } 43 | 44 | @Override 45 | public DependencyCreator output() { 46 | throw new IllegalArgumentException("Cannot use this on ConditionalStep"); 47 | } 48 | 49 | public DependencyCreator ifTrue() { 50 | return new DependencyCreator((Step) this) { 51 | @Override 52 | public void link(Step child) { 53 | getChildren().add(child); 54 | } 55 | }; 56 | } 57 | 58 | public DependencyCreator ifFalse() { 59 | return new DependencyCreator((Step) this) { 60 | @Override 61 | public void link(Step child) { 62 | falseChildren.add(child); 63 | } 64 | }; 65 | } 66 | 67 | private static class ConditionalResult { 68 | private final boolean condition; 69 | private final O output; 70 | 71 | ConditionalResult(boolean condition, O output) { 72 | this.condition = condition; 73 | this.output = output; 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/no/systek/dataflow/steps/ListStep.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow.steps; 2 | 3 | import java.util.List; 4 | import java.util.function.Consumer; 5 | 6 | import no.systek.dataflow.Step; 7 | 8 | public abstract class ListStep extends Step, O> { 9 | 10 | protected ListStep(String name) { 11 | super(name, Integer.MAX_VALUE); 12 | } 13 | 14 | @Override 15 | protected void run(List input, Consumer onResult) { 16 | //noinspection unchecked 17 | execute(input).forEach(onResult); 18 | } 19 | 20 | protected abstract List execute(List in); 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/no/systek/dataflow/steps/PairJoinStep.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow.steps; 2 | 3 | import java.util.LinkedList; 4 | import java.util.Queue; 5 | import java.util.function.Consumer; 6 | 7 | import no.systek.dataflow.PriorityTaskQueue; 8 | import no.systek.dataflow.Step; 9 | 10 | /** 11 | * Collects and sorts inputs into two internal queues and produces one joined output as soon as one of 12 | * each input type is available 13 | */ 14 | public abstract class PairJoinStep extends Step { 15 | 16 | private final Object CLEANUP = new Object(); 17 | private final Queue left; 18 | private final Queue right; 19 | private boolean cleanupScheduled; 20 | 21 | protected PairJoinStep(String name) { 22 | super(name, 1); 23 | this.left = new LinkedList<>(); 24 | this.right = new LinkedList<>(); 25 | } 26 | 27 | protected abstract O join(Ileft left, Iright right); 28 | 29 | protected abstract boolean isLeft(Object input); 30 | 31 | @Override 32 | protected void run(Object input, Consumer onResult) { 33 | if (CLEANUP.equals(input)) { 34 | cleanupScheduled = false; 35 | if (!left.isEmpty() || !right.isEmpty()) { 36 | throw new RuntimeException("Joiner step has unsatisfied dependencies, something went wrong"); 37 | } 38 | return; 39 | } 40 | 41 | if (isLeft(input)) { 42 | left.add((Ileft) input); 43 | } else { 44 | right.add((Iright) input); 45 | } 46 | 47 | if (!left.isEmpty() && !right.isEmpty()) { 48 | onResult.accept(join(left.poll(), right.poll())); 49 | } 50 | } 51 | 52 | @Override 53 | protected void afterRun(PriorityTaskQueue taskQueue) { 54 | if ((!left.isEmpty() || !right.isEmpty()) && !cleanupScheduled) { 55 | cleanupScheduled = true; 56 | taskQueue.addTask(getGraphDepth(), tq -> post(CLEANUP, tq)); 57 | } 58 | } 59 | 60 | public void dependsOnLeft(DependencyCreator left) { 61 | addParent((Step) left.step); 62 | left.link((Step) this); 63 | } 64 | 65 | public void dependsOnRight(DependencyCreator right) { 66 | addParent((Step) right.step); 67 | right.link((Step) this); 68 | } 69 | 70 | @Override 71 | public void dependsOn(DependencyCreator dependency) { 72 | throw new IllegalArgumentException( 73 | "Cannot use dependsOn() on PairJoinStep, use dependsOnLeft/dependsOnRight instead"); 74 | } 75 | } -------------------------------------------------------------------------------- /src/main/java/no/systek/dataflow/steps/SourceStep.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow.steps; 2 | 3 | import java.util.function.Consumer; 4 | 5 | import no.systek.dataflow.Step; 6 | 7 | /** 8 | * Simple step which ignores the input and produces output(s) independent from the input 9 | */ 10 | public abstract class SourceStep extends Step { 11 | 12 | public SourceStep(String name, int maxParallelExecution) { 13 | super(name, maxParallelExecution); 14 | } 15 | 16 | @Override 17 | protected void run(Object input, Consumer onResult) { 18 | onResult.accept(get()); 19 | } 20 | 21 | protected abstract O get(); 22 | } 23 | -------------------------------------------------------------------------------- /src/test/java/no/systek/dataflow/AbstractStepTest.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | 6 | import java.util.concurrent.ExecutorService; 7 | import java.util.concurrent.Executors; 8 | import java.util.concurrent.TimeUnit; 9 | 10 | public abstract class AbstractStepTest { 11 | 12 | private ExecutorService executorService; 13 | protected StepExecutor stepExecutor; 14 | 15 | @Before 16 | public void setup() { 17 | executorService = Executors.newFixedThreadPool(5); 18 | stepExecutor = new StepExecutor(executorService, s -> { 19 | }, () -> null, 5, 20, TimeUnit.SECONDS); 20 | } 21 | 22 | @After 23 | public void cleanup() { 24 | executorService.shutdown(); 25 | executorService = null; 26 | stepExecutor = null; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/test/java/no/systek/dataflow/CappuccinoTest.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow; 2 | 3 | import no.systek.dataflow.steps.CollectorStep; 4 | import no.systek.dataflow.steps.PairJoinStep; 5 | import no.systek.dataflow.types.*; 6 | import org.junit.Test; 7 | 8 | import java.util.Arrays; 9 | import java.util.List; 10 | import java.util.Random; 11 | import java.util.stream.Collectors; 12 | 13 | import static org.hamcrest.Matchers.notNullValue; 14 | import static org.hamcrest.core.Is.is; 15 | import static org.junit.Assert.assertThat; 16 | 17 | public class CappuccinoTest extends AbstractStepTest { 18 | 19 | @Test 20 | public void makeOneCappuccino() { 21 | final Random random = new Random(203); 22 | 23 | /* 24 | * The following graph of steps produces one cappuccino: 25 | * 26 | * TapWater GrindBeans FoamMilk 27 | * | | | 28 | * v | | 29 | * HeatWater <------+ | | 30 | * | | | | 31 | * v | | | 32 | * HotEnough? --no--+ | | 33 | * | | | 34 | * yes | | 35 | * | v | 36 | * +----------------> brew | 37 | * | | 38 | * v | 39 | * Cappuccino <------------+ 40 | * 41 | * credit: http://stackoverflow.com/questions/10855045/executing-dependent-tasks-in-parallel-in-java 42 | * 43 | */ 44 | 45 | TapWater tapWater = new TapWater(); 46 | FoamMilk foamMilk = new FoamMilk(); 47 | GrindBeans grindBeans = new GrindBeans(); 48 | 49 | HotEnough hotEnough = new HotEnough(); 50 | 51 | CoffeeBrewer brew = new CoffeeBrewer(); 52 | 53 | Step heatWater = Steps.newParallel(water -> 54 | work(new Water(random.nextInt(10) + 90), "Heating water")); 55 | 56 | CappuccinoStep cappuccino = new CappuccinoStep(); 57 | 58 | // Setup execution plan by setting the tasks dependencies 59 | cappuccino.dependsOnLeft(brew.output()); 60 | cappuccino.dependsOnRight(foamMilk.output()); 61 | 62 | brew.dependsOnLeft(grindBeans.output()); 63 | brew.dependsOnRight(hotEnough.ifTrue()); 64 | 65 | hotEnough.dependsOn(heatWater.output()); 66 | heatWater.dependsOn(hotEnough.ifFalse()); 67 | heatWater.dependsOn(tapWater.output()); 68 | 69 | assertThat(stepExecutor.execute(cappuccino, new Order("CappuccinoOrder")), notNullValue()); 70 | } 71 | 72 | @Test 73 | public void makeMultipleCappuccino() { 74 | final Random random = new Random(203); 75 | 76 | /* 77 | * Let's make multiple cappuccinos in parallel. The water boiler can boil water for two cups at once, so it 78 | * waits until two inputs are available. 79 | * 80 | * OrderSplitter 81 | * | 82 | * | 83 | * +--> GrindBeans-------------------------------------------+ 84 | * | | 85 | * +--> TapWater | 86 | * | | | 87 | * | v v 88 | * | Collect(2)----> HeatWater----> HotEnough?---yes---> Brew 89 | * | ^ | | 90 | * | +-------------------------no----+ | 91 | * | v 92 | * +--> FoamMilk--------------------------------------> Cappuccino 93 | * 94 | */ 95 | 96 | TapWater tapWater = new TapWater(); 97 | FoamMilk foamMilk = new FoamMilk(); 98 | GrindBeans grindBeans = new GrindBeans(); 99 | 100 | HotEnough hotEnough = new HotEnough(); 101 | 102 | CoffeeBrewer brew = new CoffeeBrewer(); 103 | Step, Water> heatWater = Steps.newParallelListStep(waters -> work( 104 | waters.stream().map(water -> new Water(random.nextInt(10) + 90)).collect(Collectors.toList()), 105 | "heating multiple waters at once (" + waters.size() + ")")); 106 | 107 | CollectorStep collector = Steps.newCollector(2); 108 | Step, Order> orderSplitter = Steps.newParallelListStep(orders -> orders); 109 | 110 | CappuccinoStep cappuccino = new CappuccinoStep(); 111 | 112 | // Setup execution plan by setting the tasks dependencies 113 | cappuccino.dependsOnLeft(brew.output()); 114 | cappuccino.dependsOnRight(foamMilk.output()); 115 | 116 | brew.dependsOnLeft(grindBeans.output()); 117 | brew.dependsOnRight(hotEnough.ifTrue()); 118 | 119 | hotEnough.dependsOn(heatWater.output()); 120 | 121 | heatWater.dependsOn(collector.output()); // water heater is shared, heats for 2 orders at the same time 122 | 123 | collector.dependsOn(hotEnough.ifFalse()); 124 | collector.dependsOn(tapWater.output()); 125 | 126 | tapWater.dependsOn(orderSplitter.output()); 127 | grindBeans.dependsOn(orderSplitter.output()); 128 | foamMilk.dependsOn(orderSplitter.output()); 129 | 130 | List cappuccinos = stepExecutor.executeList(cappuccino, Arrays.asList( 131 | new Order("Order1"), 132 | new Order("order2"), 133 | new Order("order3"))); 134 | assertThat(cappuccinos.size(), is(3)); 135 | } 136 | 137 | private static T work(T doneValue, String task) { 138 | log(task, "..."); 139 | try { 140 | Thread.sleep(1000); 141 | } catch (InterruptedException e) { 142 | throw new RuntimeException(e); 143 | } 144 | log(task, "...done"); 145 | return doneValue; 146 | } 147 | 148 | private static void log(String... msg) { 149 | System.out.println(Thread.currentThread().getName() + ": " + String.join(" ", msg)); 150 | } 151 | 152 | 153 | /* 154 | * Defining small and reusable steps for the tests 155 | */ 156 | 157 | private static class HotEnough extends Steps.SimpleConditionalStep { 158 | 159 | HotEnough() { 160 | super(null); 161 | } 162 | 163 | @Override 164 | boolean test(Water water) { 165 | boolean result = water.temperature > 95; 166 | log("Water hot enough? " + result); 167 | return result; 168 | } 169 | } 170 | 171 | private static class TapWater extends Steps.SimpleStep { 172 | 173 | TapWater() { 174 | super(null, Integer.MAX_VALUE); 175 | } 176 | 177 | @Override 178 | Water execute(Order order) { 179 | return work(new Water(18), "tapping water for " + order); 180 | } 181 | } 182 | 183 | private static class CoffeeBrewer extends PairJoinStep { 184 | CoffeeBrewer() { 185 | super(null); 186 | } 187 | 188 | @Override 189 | protected BlackCoffee join(GrindedCoffee grindedCoffee, Water water) { 190 | return work(new BlackCoffee(grindedCoffee, water), "brewing coffee"); 191 | } 192 | 193 | @Override 194 | protected boolean isLeft(Object input) { 195 | return input instanceof GrindedCoffee; 196 | } 197 | } 198 | 199 | private static class CappuccinoStep extends PairJoinStep { 200 | CappuccinoStep() { 201 | super(null); 202 | } 203 | 204 | @Override 205 | protected Cappuccino join(BlackCoffee blackCoffee, FoamedMilk foamedMilk) { 206 | return work(new Cappuccino(blackCoffee, foamedMilk), "making cappuccino"); 207 | } 208 | 209 | @Override 210 | protected boolean isLeft(Object input) { 211 | return input instanceof BlackCoffee; 212 | } 213 | } 214 | 215 | private final class GrindBeans extends Steps.SimpleStep { 216 | GrindBeans() { 217 | super(null, Integer.MAX_VALUE); 218 | } 219 | 220 | @Override 221 | GrindedCoffee execute(Order order) { 222 | return work(new GrindedCoffee(order), "grinding coffee for " + order); 223 | } 224 | } 225 | 226 | private final class FoamMilk extends Steps.SimpleStep { 227 | FoamMilk() { 228 | super(null, Integer.MAX_VALUE); 229 | } 230 | 231 | @Override 232 | FoamedMilk execute(Order order) { 233 | return work(new FoamedMilk(order), "foaming milk for " + order); 234 | } 235 | } 236 | 237 | } 238 | -------------------------------------------------------------------------------- /src/test/java/no/systek/dataflow/PriorityTaskQueueTest.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow; 2 | 3 | import static org.hamcrest.Matchers.is; 4 | import static org.junit.Assert.assertThat; 5 | 6 | import java.util.LinkedList; 7 | import java.util.Queue; 8 | import java.util.concurrent.CountDownLatch; 9 | import java.util.concurrent.ExecutorService; 10 | import java.util.concurrent.Executors; 11 | import java.util.concurrent.TimeUnit; 12 | import java.util.concurrent.atomic.AtomicBoolean; 13 | import java.util.concurrent.atomic.AtomicInteger; 14 | import java.util.concurrent.atomic.AtomicReference; 15 | import java.util.function.Consumer; 16 | 17 | import org.junit.Test; 18 | 19 | public class PriorityTaskQueueTest { 20 | 21 | @Test 22 | public void testParallelExecutionAtPriority1() { 23 | final int parallelTasks = 10; 24 | PriorityTaskQueue pq = new PriorityTaskQueue(parallelTasks, () -> null, s -> { 25 | }); 26 | 27 | AtomicInteger successCounter = new AtomicInteger(); 28 | CountDownLatch c = new CountDownLatch(parallelTasks); 29 | 30 | for (int i = 0; i < parallelTasks; i++) { 31 | pq.addTask(2, e -> { 32 | c.countDown(); 33 | try { 34 | if (c.await(2, TimeUnit.SECONDS)) { 35 | successCounter.incrementAndGet(); 36 | } 37 | } catch (InterruptedException ignored) { 38 | } 39 | }); 40 | } 41 | 42 | ExecutorService executorService = Executors.newCachedThreadPool(); 43 | Queue exceptions = new LinkedList<>(); 44 | assertThat(pq.executeTasksAndAwaitDone(executorService, exceptions::offer, 1, TimeUnit.SECONDS), 45 | is(true)); 46 | assertThat(exceptions.size(), is(0)); 47 | assertThat(successCounter.get(), is(parallelTasks)); 48 | executorService.shutdown(); 49 | } 50 | 51 | @Test 52 | public void testAllTasksAtPriority1AreDoneFirst() { 53 | final int parallelTasks = 10; 54 | PriorityTaskQueue pq = new PriorityTaskQueue(parallelTasks, () -> null, s -> { 55 | }); 56 | 57 | final AtomicInteger priority1Started = new AtomicInteger(parallelTasks); 58 | final AtomicInteger errorCounter = new AtomicInteger(); 59 | Consumer prio1Task = e -> priority1Started.decrementAndGet(); 60 | Consumer prio2Task = e -> { 61 | if (priority1Started.get() == parallelTasks) { 62 | errorCounter.incrementAndGet(); 63 | } 64 | }; 65 | 66 | for (int i = 0; i < parallelTasks; i++) { 67 | pq.addTask(2, prio2Task); 68 | pq.addTask(1, prio1Task); 69 | } 70 | 71 | ExecutorService executorService = Executors.newCachedThreadPool(); 72 | Queue exceptions = new LinkedList<>(); 73 | assertThat( 74 | pq.executeTasksAndAwaitDone(executorService, exceptions::offer, 1, TimeUnit.SECONDS), 75 | is(true)); 76 | assertThat(exceptions.size(), is(0)); 77 | assertThat(errorCounter.get(), is(0)); 78 | executorService.shutdown(); 79 | } 80 | 81 | @Test 82 | public void testMovingBackToHigherPriority() { 83 | final int parallelTasks = 10; 84 | PriorityTaskQueue pq = new PriorityTaskQueue(parallelTasks, () -> null, s -> { 85 | }); 86 | 87 | AtomicBoolean test = new AtomicBoolean(false); 88 | AtomicInteger errors = new AtomicInteger(); 89 | AtomicInteger doneCounter = new AtomicInteger(); 90 | pq.addTask(10, q1 -> { 91 | q1.addTask(9, q2 -> { 92 | q2.addTask(7, q3 -> { 93 | if (!test.compareAndSet(false, true)) { 94 | errors.incrementAndGet(); 95 | } 96 | doneCounter.incrementAndGet(); 97 | }); 98 | q2.addTask(8, q3 -> { 99 | if (!test.compareAndSet(true, false)) { 100 | errors.incrementAndGet(); 101 | } 102 | doneCounter.incrementAndGet(); 103 | }); 104 | doneCounter.incrementAndGet(); 105 | }); 106 | doneCounter.incrementAndGet(); 107 | }); 108 | 109 | ExecutorService executorService = Executors.newCachedThreadPool(); 110 | Queue exceptions = new LinkedList<>(); 111 | assertThat( 112 | pq.executeTasksAndAwaitDone(executorService, exceptions::offer, 1, TimeUnit.SECONDS), 113 | is(true)); 114 | assertThat(exceptions.size(), is(0)); 115 | assertThat(errors.get(), is(0)); 116 | assertThat(doneCounter.get(), is(4)); 117 | executorService.shutdown(); 118 | } 119 | 120 | @Test 121 | public void korrelasjonsIdIsPreserved() { 122 | final int parallelTasks = 10; 123 | 124 | String korrelatjonsId = "TEST123"; 125 | ThreadLocal korrelasjonsId = new ThreadLocal<>(); 126 | korrelasjonsId.set(korrelatjonsId); 127 | PriorityTaskQueue pq = new PriorityTaskQueue(parallelTasks, korrelasjonsId::get, korrelasjonsId::set); 128 | 129 | AtomicReference capcturedKorrelatjonsId = new AtomicReference<>(); 130 | pq.addTask(1, q -> capcturedKorrelatjonsId.set(korrelasjonsId.get())); 131 | 132 | ExecutorService executorService = Executors.newCachedThreadPool(); 133 | Queue exceptions = new LinkedList<>(); 134 | assertThat( 135 | pq.executeTasksAndAwaitDone(executorService, exceptions::offer, 1, TimeUnit.SECONDS), 136 | is(true)); 137 | assertThat(exceptions.size(), is(0)); 138 | assertThat(capcturedKorrelatjonsId.get(), is(korrelatjonsId)); 139 | executorService.shutdown(); 140 | 141 | } 142 | 143 | @Test 144 | public void testTimeout() { 145 | PriorityTaskQueue pq = new PriorityTaskQueue(10, () -> null, s -> { 146 | }); 147 | pq.addTask(2, q -> { 148 | while (true) { 149 | try { 150 | Thread.sleep(2000); 151 | break; 152 | } catch (InterruptedException ignored) { 153 | } 154 | } 155 | }); 156 | ExecutorService executorService = Executors.newCachedThreadPool(); 157 | Queue exceptions = new LinkedList<>(); 158 | assertThat( 159 | pq.executeTasksAndAwaitDone(executorService, exceptions::offer, 1, TimeUnit.SECONDS), 160 | is(false)); 161 | assertThat(exceptions.size(), is(0)); 162 | executorService.shutdown(); 163 | 164 | } 165 | 166 | } -------------------------------------------------------------------------------- /src/test/java/no/systek/dataflow/StepTest.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow; 2 | 3 | import no.systek.dataflow.steps.PairJoinStep; 4 | import no.systek.dataflow.steps.SourceStep; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import java.util.HashSet; 9 | import java.util.LinkedList; 10 | import java.util.List; 11 | import java.util.concurrent.atomic.AtomicInteger; 12 | import java.util.function.Consumer; 13 | import java.util.stream.IntStream; 14 | 15 | import static org.hamcrest.core.Is.is; 16 | import static org.junit.Assert.assertThat; 17 | 18 | public class StepTest extends AbstractStepTest { 19 | 20 | @Test 21 | public void treeIsConfiguredCorrectly() { 22 | List> steps = new LinkedList<>(); 23 | 24 | IntStream.range(1, 10).forEach(level -> { 25 | steps.add(createStep(level)); 26 | if (steps.size() > 1) { 27 | steps.get(steps.size() - 1).dependsOn(steps.get(steps.size() - 2).output()); 28 | } 29 | }); 30 | 31 | HashSet> roots = new HashSet<>(); 32 | steps.get(steps.size() - 1).configureTreeAndFindRoots(new HashSet<>(), roots); 33 | 34 | assertThat(roots.size(), is(1)); 35 | AtomicInteger assertLevel = new AtomicInteger(1); 36 | steps.forEach(s -> { 37 | int expectedDepth = assertLevel.getAndIncrement(); 38 | assertThat(s.getName(), is(String.valueOf(expectedDepth))); 39 | assertThat(s.getGraphDepth(), is(expectedDepth)); 40 | }); 41 | } 42 | 43 | @Test 44 | public void sourceStepTest() { 45 | Assert.assertThat(stepExecutor.execute(Steps.newSource(() -> "Hello world")), is("Hello world")); 46 | } 47 | 48 | @Test 49 | public void singleStepTest() { 50 | Assert.assertThat(stepExecutor.execute(Steps.newSingle(in -> in + 1), 1), is(2)); 51 | } 52 | 53 | @Test 54 | public void joinerTest() { 55 | PairJoinStep joinStep = Steps.newJoiner("Hello"::equals, (left, right) -> left + " " + right); 56 | 57 | joinStep.dependsOnLeft(Steps.newSource(() -> "Hello").output()); 58 | joinStep.dependsOnRight(Steps.newSource(() -> "World").output()); 59 | 60 | assertThat(stepExecutor.execute(joinStep), is("Hello World")); 61 | } 62 | 63 | @Test 64 | public void conditionTest() { 65 | Steps.SimpleConditionalStep condition = Steps.newCondition("Hello"::equals); 66 | Step sink = new Step(1) { 67 | @Override 68 | protected void run(String input, Consumer onResult) { 69 | } 70 | }; 71 | Step tail = Steps.newSingle(input -> input); 72 | 73 | tail.dependsOn(condition.ifTrue()); 74 | sink.dependsOn(condition.ifFalse()); 75 | condition.dependsOn(Steps.newSource(() -> "Hello").output()); 76 | condition.dependsOn(Steps.newSource(() -> "World").output()); 77 | 78 | assertThat(stepExecutor.execute(tail), is("Hello")); 79 | } 80 | 81 | private Step createStep(int level) { 82 | return new SourceStep(String.valueOf(level), Integer.MAX_VALUE) { 83 | @Override 84 | protected Object get() { 85 | return null; 86 | } 87 | }; 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/test/java/no/systek/dataflow/types/BlackCoffee.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow.types; 2 | 3 | public class BlackCoffee { 4 | 5 | public final GrindedCoffee grindedCoffee; 6 | public final Water water; 7 | 8 | public BlackCoffee(GrindedCoffee grindedCoffee, Water water) { 9 | this.grindedCoffee = grindedCoffee; 10 | this.water = water; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/test/java/no/systek/dataflow/types/Cappuccino.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow.types; 2 | 3 | public class Cappuccino { 4 | public final BlackCoffee blackCoffee; 5 | public final FoamedMilk foamedMilk; 6 | 7 | public Cappuccino(BlackCoffee blackCoffee, FoamedMilk foamedMilk) { 8 | this.blackCoffee = blackCoffee; 9 | this.foamedMilk = foamedMilk; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/test/java/no/systek/dataflow/types/FoamedMilk.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow.types; 2 | 3 | public class FoamedMilk { 4 | private final Order order; 5 | 6 | public FoamedMilk(Order order) { 7 | this.order = order; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/test/java/no/systek/dataflow/types/GrindedCoffee.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow.types; 2 | 3 | public class GrindedCoffee { 4 | private final Order order; 5 | 6 | public GrindedCoffee(Order order) { 7 | this.order = order; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/test/java/no/systek/dataflow/types/Order.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow.types; 2 | 3 | public class Order { 4 | private final String name; 5 | 6 | public Order(String name) { 7 | this.name = name; 8 | } 9 | 10 | @Override 11 | public String toString() { 12 | return "Order{" + 13 | "name='" + name + '\'' + 14 | '}'; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/test/java/no/systek/dataflow/types/Water.java: -------------------------------------------------------------------------------- 1 | package no.systek.dataflow.types; 2 | 3 | public class Water { 4 | 5 | public final int temperature; 6 | 7 | public Water(int temperature) { 8 | this.temperature = temperature; 9 | } 10 | } 11 | --------------------------------------------------------------------------------