├── .gitignore ├── License.md ├── README.md ├── pom.xml └── src ├── main └── java │ └── org │ └── reactivejournal │ ├── examples │ ├── fastproducerslowconsumer │ │ ├── FastProducerSlowConsumer.java │ │ ├── RxJavaBackPressure.java │ │ ├── RxJournalBackPressureBuffer.java │ │ ├── RxJournalBackPressureLatest.java │ │ ├── RxJournalBackPressureTestingFasterConsumer.java │ │ └── resources │ │ │ └── 20170525.cq4 │ └── helloworld │ │ ├── BytesToWordsProcessor.java │ │ ├── HelloWorld.java │ │ ├── HelloWorldApp_JounalAsObserver.java │ │ ├── HelloWorldApp_JournalPlayThrough.java │ │ ├── HelloWorldRemote.java │ │ ├── HelloWorldTest.java │ │ └── HelloWorldUsingRxJava.java │ ├── impl │ ├── DataItemProcessor.java │ ├── EndOfStream.java │ ├── PlayOptions.java │ ├── ReactiveJournal.java │ ├── ReactivePlayer.java │ ├── ReactiveRecorder.java │ ├── ReactiveStatus.java │ ├── ReactiveValidator.java │ ├── ValidationResult.java │ └── rxjava │ │ └── RxJavaPlayer.java │ └── util │ ├── DSUtil.java │ └── TriConsumer.java ├── playground └── java │ └── queue │ ├── MarketData.java │ ├── MarshallableHolder.java │ ├── PauseTest.java │ ├── QueuePlayGround.java │ └── TestStore.java └── test ├── java └── org │ └── reactivejournal │ └── impl │ ├── ReactiveErrorTest.java │ ├── ReactiveFromUntilSeqNoTest.java │ ├── ReactiveFromUntilTimeTest.java │ ├── ReactiveJournalDirectoryTest.java │ ├── ReactivePlayerTest.java │ ├── ReactiveRecorderTest.java │ ├── ReactiveReplayRateTest.java │ └── ReactiveRequestTest.java └── resources ├── 20170612.cq4 └── playTest.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 2 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 3 | 4 | # User-specific stuff: 5 | .idea/**/workspace.xml 6 | .idea/**/tasks.xml 7 | .idea/dictionaries 8 | *.iml 9 | todolist.txt 10 | 11 | # Sensitive or high-churn files: 12 | .idea/**/dataSources/ 13 | .idea/**/dataSources.ids 14 | .idea/**/dataSources.xml 15 | .idea/**/dataSources.local.xml 16 | .idea/**/sqlDataSources.xml 17 | .idea/**/dynamic.xml 18 | .idea/**/uiDesigner.xml 19 | 20 | # Gradle: 21 | .idea/**/gradle.xml 22 | .idea/**/libraries 23 | 24 | # CMake 25 | cmake-build-debug/ 26 | 27 | # Mongo Explorer plugin: 28 | .idea/**/mongoSettings.xml 29 | 30 | ## File-based project format: 31 | *.iws 32 | 33 | ## Plugin-specific files: 34 | 35 | # IntelliJ 36 | /out/ 37 | 38 | # mpeltonen/sbt-idea plugin 39 | .idea_modules/ 40 | 41 | # JIRA plugin 42 | atlassian-ide-plugin.xml 43 | 44 | # Cursive Clojure plugin 45 | .idea/replstate.xml 46 | 47 | # Crashlytics plugin (for Android Studio and IntelliJ) 48 | com_crashlytics_export_strings.xml 49 | crashlytics.properties 50 | crashlytics-build.properties 51 | fabric.properties -------------------------------------------------------------------------------- /License.md: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ReactiveJournal 2 | 3 | ReactiveJournal supports [Reactive](http://www.reactive-streams.org/) libraries by adding 4 | functionality to record and replay reactive streams. 5 | 6 | ## Downloading the project 7 | 8 | ### Maven 9 | ReactiveJournal is a Maven project so you can `git clone` the project and build in the usual way. 10 | 11 | The intention is for this project to make its way into Maven Central (work in progress). 12 | 13 | ### Download the jar 14 | Go to the releases section of the project. With each release there will be an uber jar that you 15 | can download with the ReactiveJournal classes and all dependencies. 16 | 17 | Once downloaded you can test that it works by running: 18 | 19 | ```` 20 | java -cp ./reactivejournal-x.x.x.jar org.reactivejournal.examples.helloworld.HelloWorld 21 | ```` 22 | ## Why ReactiveJournal?? 23 | 24 | Picture this scenario... 25 | 26 | You've just released your shiny new Reactive project. Streams of data start to flow into the system, 27 | your 'Reactives' are busy processing the data, enriching it, making decisions based on it, creating 28 | new streams of data for Reactives further down the line to consume. It's the well oiled engine of 29 | an F1 car, a 30 | thing of beauty, poetry in motion. Exactly what a Reactive system was designed to do. 31 | 32 | But then... 33 | 34 | Something starts going wrong. Perhaps data received isn't quite as clean as it should be, 35 | the system gets dirty, components clog up and fail, data backs up. Processes slow down, 36 | before long we have total melt down. Our F1 engine is a heap a smouldering metal. 37 | 38 | But what went wrong... this is what we need to be able to our system: 39 | 40 | * Recreate the exact situation so that can step through it and understand 41 | where the root of the problem is. 42 | 43 | * Capture that scenario fix it and include it in a test case so that it never 44 | happens again. 45 | 46 | * Monitor the slowdown of the system by observing the queues build up. Recover 47 | the system without losing any data in the meantime. 48 | 49 | * Deal with the surge in data that caused the component to start runing 50 | out of memory. 51 | 52 | * Run multiple processes to process our data rather than relying on the one 53 | which went over. 54 | 55 | * All the above without slowing the system down or having to do any development. 56 | 57 | Journaling addresses all these points. ReactiveJournal was designed to makes journaling for 58 | Reactive programs as simple as it could possible be. 59 | 60 | 61 | ## Primary Motivations Behind ReactiveJournal 62 | 63 | ### 1. Testing 64 | 65 | Testing is a primary motivation for ReactiveJournal. ReactiveJournal allows developers to 66 | black box test their code by recording all inputs and outputs in and out of their programs. 67 | 68 | An obvious use case are unit tests where ReactiveJournal recordings can be used to create 69 | comprehensive tests (see [HelloWorldTest](https://github.com/danielshaya/reactivejournal/blob/master/src/main/java/org/reactivejournal/examples/helloworld/HelloWorldTest.java) for an example). This example makes use of 70 | `ReactiveValidator` which allows unit tests to compare their results against previously 71 | recorded results in the journal. 72 | 73 | Another powerful use case is to enable users to replay production data into test systems. 74 | By simply copying over the journal file from a production system and replaying 75 | all or part of the file 76 | into a test system the exact conditions of the primary system will be reproduced. 77 | 78 | ### 2. Remote Connections 79 | 80 | ReactiveJournal can be recorded on one JVM and can be replayed (in real-time if required) 81 | on one or more 82 | JVMs provided they have access to the journal file location. 83 | 84 | The remote connection can either read from the beginning of the recording or just start with live 85 | updates from the recorder. The remote connection (the 'listener') can optionally write back to the 86 | journal effecting a two way conversation or RPC. There can be multiple readers and writers to the 87 | journal. 88 | 89 | ReactiveJournal uses [Chronicle-Queue](https://github.com/OpenHFT/Chronicle-Queue/blob/master/README.adoc) (a memory mapped file solution) 90 | serialisation meaning that 91 | the process of moving data from one JVM to another is exceedingly efficient and can be achieved 92 | in single digit micro seconds. 93 | 94 | If you need to pass data between JVMs on the same machine this is not only the most efficient way 95 | to do so but you will also provide you with a full recording of the data that is 96 | transferred between the JVMs. 97 | 98 | Setting up remote listeners allows a system to have live (hot) backup processes. Another 99 | powerful strategy is to set 100 | up processes running different code bases so that you can live test new code or play with 101 | alternative algorithms and compare results with the exisiting sytem. 102 | 103 | ### 3. Slow consumers (handling back pressure) 104 | 105 | If you have a fast producer that you can't slow down but your consumer can't keep up 106 | there are a few options available to your system. 107 | 108 | Most often you end up implementing strategies that hold buffers of data in memory allowing the 109 | consumer to catch up eventually. The problem with those sort of strategies are one, if your process 110 | crashes you lose all the data in your buffer. Therefore if you need to consume the fast data in a 111 | transactional manner this will not be an option. Two, you may run out of memory if the 112 | buffers get really big. At the very least you will probably need to run your JVM with a large 113 | memory setting that many be inefficient. For latency sensitive applications it will 114 | put pressure on the GC which will not be acceptable. 115 | 116 | In such scenarios you can publish your data to ReactiveJournal which shouldn't have any problems 117 | keeping up with most feeds (typically writing in the order of 1,000,000/s). You can then read 118 | your data from ReactiveJournal knowing that the data is safely stored on disk without having to 119 | keep any data in memory buffers. 120 | 121 | See more about this topic below in the [Examples](https://github.com/danielshaya/reactivejournal/tree/master/src/main/java/org/reactivejournal/examples) section 122 | 123 | ## Design Goals 124 | 125 | - Recording to the journal is transactional i.e. no data will be lost if the 126 | program crashes 127 | - Recording and playback is so fast that it won't slow down the host program. 128 | - Recording and playback can be achieved without any gc overhead 129 | - RxRecorder can be easily added (or even retro-fitted) into any `Reactive` project 130 | 131 | # Quick Start 132 | ## Creating a Journal 133 | 134 | An `ReactiveJournal` is created as follows: 135 | 136 | ReactiveJournal reactiveJournal = new ReactiveJournal(String dir); 137 | 138 | The directory is the location where the serialised file will be created 139 | 140 | ## Recording a reactive stream 141 | `ReactiveRecorder` allows any Reactive [`Publisher`](https://github.com/reactive-streams/reactive-streams-jvm/blob/master/api/src/main/java/org/reactivestreams/Publisher.java) to be journaled to disk using 142 | the `record` function: 143 | 144 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 145 | reactiveRexcorder.record(Publisher) 146 | 147 | For notes on threading see FAQ below. 148 | 149 | ## Playing back a reactive stream 150 | 151 | `ReactivePlayer` is used to playback the journal recording: 152 | 153 | ReactivePlayer reactivePlayer = reactiveJournal.createReactivePlayer(); 154 | Publisher = reactivePlayer.play(new PlayOptions()); 155 | 156 | There are a number of options that can be configured using `PlayOptions`. These 157 | include filtering the stream by time and stream. Playback speed can also be 158 | controlled using this configuration. 159 | 160 | ## Viewing the contents of a journal 161 | 162 | `ReactiveJournal` is created and stored to disk using the low latency Chronicle-Queue library. 163 | The data can be examined in plain ASCII using the writeToDisk function: 164 | 165 | reactiveJournal.writeToDisk(String fileName, boolean printToSdout) 166 | 167 | ## Putting it together with HelloWorld 168 | 169 | Full code example code [HelloWorldApp](https://github.com/danielshaya/ReactiveJournal/blob/master/src/main/java/org/ReactiveJournal/examples/helloworld/HelloWorld.java). 170 | 171 | ```java 172 | package org.reactivejournal.examples.helloworld; 173 | 174 | import org.reactivejournal.impl.PlayOptions; 175 | import org.reactivejournal.impl.ReactiveJournal; 176 | import org.reactivejournal.impl.ReactiveRecorder; 177 | import org.reactivestreams.Publisher; 178 | import org.reactivestreams.Subscriber; 179 | import org.reactivestreams.Subscription; 180 | 181 | import java.io.File; 182 | import java.io.IOException; 183 | 184 | /** 185 | * Simple Demo Program 186 | */ 187 | public class HelloWorld { 188 | private static final String tmpDir = System.getProperty("java.io.tmpdir"); 189 | 190 | public static void main(String[] args) throws IOException { 191 | //1. Create the reactiveRecorder and delete any previous content by clearing the cache 192 | ReactiveJournal reactiveJournal = new ReactiveJournal(tmpDir + File.separator + "HW"); 193 | reactiveJournal.clearCache(); 194 | 195 | //2. Create a HelloWorld Publisher 196 | Publisher helloWorldFlowable = subscriber -> { 197 | subscriber.onNext("Hello World!"); 198 | subscriber.onComplete(); 199 | }; 200 | 201 | //3. Pass the Publisher into the reactiveRecorder which will subscribe to it and record all events. 202 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 203 | reactiveRecorder.record(helloWorldFlowable, ""); 204 | 205 | //4. Subscribe to ReactiveJournal and print out results 206 | Publisher recordedObservable = reactiveJournal.createReactivePlayer().play(new PlayOptions()); 207 | 208 | recordedObservable.subscribe(new Subscriber() { 209 | @Override 210 | public void onSubscribe(Subscription subscription) { 211 | subscription.request(Long.MAX_VALUE); 212 | } 213 | 214 | @Override 215 | public void onNext(Object o) { 216 | System.out.println(o); 217 | } 218 | 219 | @Override 220 | public void onError(Throwable throwable) { 221 | System.err.println(throwable); 222 | } 223 | 224 | @Override 225 | public void onComplete() { 226 | System.out.println("Hello World Complete"); 227 | } 228 | }); 229 | 230 | //5. Sometimes useful to see the recording written to a file 231 | reactiveJournal.writeToFile(tmpDir + File.separator + "/hw.txt",true); 232 | } 233 | } 234 | ``` 235 | The results of running this program can be seen below: 236 | 237 | ```` 238 | [main] INFO org.reactivejournal.impl.ReactiveJournal - Deleting existing recording [/tmp/HW/.reactiveJournal] 239 | Hello World! 240 | [main] INFO org.reactivejournal.impl.ReactiveJournal - Writing recording to dir [/tmp//hw.txt] 241 | Hello World Complete 242 | [main] INFO org.reactivejournal.impl.ReactiveJournal - VALID 1 1499202194782 Hello World! 243 | [main] INFO org.reactivejournal.impl.ReactiveJournal - COMPLETE 2 1499202194784 EndOfStream{} 244 | [main] INFO org.reactivejournal.impl.ReactiveJournal - Writing to dir complete 245 | ```` 246 | 247 | ## PlayOptions 248 | 249 | When playing back from ReactiveJournal there are a number of options that can be set summarised in the table below: 250 | They are explained more fully in the FAQ and in the [javadoc](https://github.com/danielshaya/reactivejournal/blob/master/src/main/java/org/reactivejournal/impl/PlayOptions.java) for `PlayOptions` 251 | 252 | | Option Name | Values | Default | Description | 253 | | ------------- |------------------| ------------- | -------------| 254 | | filter | any string | n/a | All messages are written with a filter tag | 255 | | pauseStrategy | SPIN / YIELD | YIELD | Determines the pause strategy if no events in the journal | 256 | | playFromNow | true / false | false | Only play message from now - ignore older messages | 257 | | playFromSeqNo | any long | Long.MIN_VALUE| Only play messages from seqNo | 258 | | playUntilSeqNo| any long | Long.MAX_VALUE| Only play messages until seqNo and then complete | 259 | | playFromTime | any long | Long.MIN_VALUE| Only play messages from time in millis | 260 | | playUntilTime | any long | Long.MAX_VALUE| Only play messages until time in millis and then complete | 261 | | replayRate | FAST / ACTUAL_TIME | FAST | Play back messages fast or with the intervals they were recorded | 262 | | sameThread | true / false | false | Play back messages on the subscriber thread or a new one | 263 | | using | any Object | null | Use this Object to populate the message - save on GC | 264 | 265 | 266 | ## FAQ 267 | 268 | ### What types of data can be serialised by ReactiveJournal 269 | 270 | Items that can be serialised to ReactiveJournal are those that can be serialised to 271 | Chronicle-Queue. 272 | 273 | These are: 274 | * AutoBoxed primitives, Strings and byte[] 275 | * Classes implementing `Serialisable` 276 | * Classes implementing `Externalizable` 277 | * Classes implementing `Marshallable` 278 | 279 | See [Chronicle Queue Docs](https://github.com/OpenHFT/Chronicle-Queue#restrictions-on-topics-and-messages) for full documentation 280 | 281 | ### How to use in conjunction with Reactive implementations such as RxJava and React? 282 | 283 | `ReactiveJava` has been designed be used with any [Reactive](https://github.com/reactive-streams) implentation 284 | such as [RxJava](https://github.com/ReactiveX/RxJava) and [Reactor](https://projectreactor.io/). 285 | 286 | For example if you had an RxJava `Flowable`, because `Flowable` implements `Publisher`, 287 | you could use the `Flowable` as input to 288 | `ReactiveRecorder.record()` which takes a `Publisher`. ()If you have an RxJava 289 | `Observable` you would have to convert it to a `Flowable` with `Observable.toFlowable()` 290 | first.) 291 | 292 | `ReactivePlayer` returns a `Publisher` that can be converted to `Flowable` with 293 | `Flowable.fromPublisher(reactivePlayer.play(options))`. There is utility class 294 | `RxPlayer` that does this for the RxJava user. 295 | 296 | 297 | ### ReactiveJournal on the critical path or as another subscriber 298 | 299 | There are 2 ways you might want to set up your `ReactiveJournal`. 300 | 301 | 1. Record your `Publisher` input into `ReactiveJournal` and then subscribe to 302 | `ReactiveJournal` for its stream of events. This effectively inserts `ReactiveJournal` 303 | into the critical path of 304 | your program. This will certainly be the setup if you are using ReactiveJournal 305 | to handle back pressure. 306 | This is demonstrated in the example program [HelloWorldApp_JournalPlayThrough](https://github.com/danielshaya/reactivejournal/blob/master/src/main/java/org/reactivejournal/examples/helloworld/HelloWorldApp_JournalPlayThrough.java) 307 | 308 | 2. Have `ReactiveJournal` as a second subscriber to your `Publisher`. This has the benefit 309 | of keeping all functions on the same thread. This might be the setup if you are using `ReactiveJournal` 310 | to record data for testing purposes. If you are using RxJava you might want to use 311 | the `ConnectableObservable` paradigm 312 | as you won't want ReactiveRecorder kicking off the connection until 313 | all the other connections have been setup. 314 | This is demonstrated in the example program [HelloWorldApp_JounalAsObserver](https://github.com/danielshaya/reactivejournal/blob/master/src/main/java/org/reactivejournal/examples/helloworld/HelloWorldApp_JounalAsObserver.java) 315 | 316 | ### Play the stream in actual time or fast 317 | 318 | The `RxPlayer` can `play` in two modes: 319 | * `ACTUAL_TIME` This plays back the stream preserving the time gaps between the events. This is 320 | important for back testing and reproducing exact conditions in unit tests. 321 | * `FAST` This plays the events as soon as they are received. This mode should be set when you are using 322 | ReactiveJournal for remote connections or when using RxJounal to deal with back pressure. 323 | 324 | ### Can ReactiveJournal be used in a low latency environment 325 | 326 | The intention is for `ReactiveJournal` to support low latency programs. The two main features to allow 327 | for this are: 328 | * Dedicating a CPU core to RxPlayer by using the `PlayOptions.setPauseStrategy(PauseStrategy.SPIN)` 329 | setting so that we don't have any context switching on the thread reading from the journal. 330 | * Setting the `PlayOptions.using()` so that there is no allocation for new events. This should enable 331 | programs to be written that have minimal GC impact, critical for reliable low latency. 332 | 333 | ## Examples 334 | 335 | There are few core example applications in the code that work through the typical 336 | use cases and are worth considering in more detail. 337 | 338 | ### HelloWorldApp_JournalPlayThrough 339 | 340 | This program demonstrates how to set up a simple 'play through' example. 341 | 342 | We have an input `Flowable` with a stream of `Byte`s. These are recorded in the journal 343 | by `ReactiveRecorder`. 344 | 345 | We then subscribe to `ReactiveJournal` with `ReactivePlayer` giving us an `Flowable` of `Byte`s 346 | which are processed by the `BytesToWordsProcessor`. The output of the processor is 347 | also recorded into `ReactiveJournal` so we have a full record of all our input and outputs to 348 | the program. 349 | 350 | Note that we use `recordAsync` rather than `record` because otherwise we would 351 | block the main thread until all the event stream had completed recording and only 352 | then would we proceed to process the items. Although in this trivial example 353 | it's hard to see the effect this has I encourage you to play with the `INTERVAL_MS` 354 | setting to see what happens as you increase the delay to something noticeable. 355 | Then try and change `recordAsync` to `async` and you will see the effect of 356 | the threading. 357 | 358 | We then display the results of the program to stdout as well as writing to a file. 359 | 360 | This recording will be valuable when it comes to writing a unit test for 361 | `BytesToWordsProcessor` which we'll see in another example. 362 | 363 | ### HelloWorldApp_JournalAsObserver 364 | 365 | This is very similar to the last example except that we processes everything 366 | on the same thread. We can do this because rather than the `BytesToWordsProcessor` 367 | subscribing to `ReactiveJournal` it subscribes directly to the `Observable` input. 368 | 369 | This is a less intrusive way to insert RxRecorder into your project but of 370 | course will not handle the back pressure problem. 371 | 372 | ### HelloWorldTest 373 | 374 | This example demonstrates how to use RxRecorder in a unit test. The journal file 375 | we created in the previous examples is used as input to test the `BytesToWordsProcessor`. 376 | The results of `BytesToWordsProcessor` are fed into `ReactiveValidator` which compares 377 | the output to the output which was recorded in the journal reporting any 378 | differences. 379 | 380 | We have effectively black boxed the inputs and outputs to `BytesToWordsProcessor` and can 381 | be confident that any changes we make to the processor will not break the existing 382 | behaviour. 383 | 384 | ### HelloWorldRemote 385 | 386 | This example is designed to show how ReactiveJournal can be used to tranfer data between JVMs. 387 | 388 | Start `HelloWorldApp_JournalPlayThrough` but increase the `INTERVAL_MS` to 1000. Then 389 | run `HelloWorldRemote`. 390 | 391 | `HelloWorldRemote` has been configured with this option: 392 | 393 | ```` java 394 | new PlayOptions().filter(HelloWorldApp_JounalAsObserver.INPUT_FILTER).playFromNow(true); 395 | ```` 396 | 397 | The playFromNow means that it will only consume current events and depending on how long 398 | a gap you have between starting the 2 programs you will see output which looks something 399 | like this: 400 | 401 | 402 | 403 | ### Fast Producer Slow Consumer 404 | 405 | In these example programs we deal with the situation where we find ourselves with a 406 | fast producer and slow consumer. 407 | 408 | In all these example we setup a scenario in [`FastProducerSlowConsumer`]() where the 409 | producer emits `Long` values every millisecond. We also create a `Consumer` which 410 | processes the Long values with a variable delay which is significantly slower 411 | than the rate that they are being produced. 412 | 413 | In other words we have the classic Fast Producer Slow Consumer scenario which needs 414 | to be handled by applying back pressure. 415 | 416 | The following example programs have all been written to 'solve' the back pressure 417 | problem we have created. 418 | 419 | #### RxJavaBackPressure 420 | 421 | Firstly let's consider how RxJava handles back pressure out of the box. 422 | 423 | A quick reminder, in RxJava2 the code was split into 2 sections: 424 | * `Observable` - no back pressure. Use when back pressure is not an issue because the 425 | code is more efficient not having to deal with this complication. 426 | * `Flowable` - handles back pressure. Use when you have to address the back pressure issue. 427 | 428 | Clearly we will only be looking at the `Flowable` part of RxJava2 in this example. 429 | 430 | This example program demonstrates how the 5 `BackpressureStrategy` modes handle back 431 | pressure. 432 | * `BUFFER` this will, as its name implies, hold the items in an in-memory buffer waiting 433 | for availablility on the consumer to process them. This is good choice for handling spikes 434 | in event traffic where the consumer will eventually be able to catch up with the 435 | producer. The problems using this strategy are: 436 | 437 | * If the program crashes the events in the buffer will be lost. Even if the 438 | program terminates normally careful attention has o be paid to draining the buffer. 439 | * If the queue builds up too much the JVM will run out of memory and crash. 440 | * It forces the program to run with a large memory setting to hold the buffer 441 | which can be a problem for programs where latency is an issue especially coupled 442 | with the next point. 443 | * The program will not be able to be designed in an allocation-free manner. Every 444 | item will have to be created in a 'new' object which will then put pressure on the GC. 445 | 446 | * `LATEST` and `DROP` deal with back pressure by making the slow consumer keep up with 447 | the fast producer. This is done by dropping events from the stream. This is a good choice 448 | where events on the stream are replaceable and you don't need to process every item. The 449 | problems with this straegy are: 450 | 451 | * If you want to back test your program against all the values in the stream to see 452 | if you might get better results by processing more events. 453 | * As with buffer you can't write GC friendly code. 454 | 455 | * `ERROR` and `MISSING` deal with back pressure by putting the program into an error state 456 | as soon as back pressure is encountered. This is useful when you don't expect any back pressure 457 | and you want the program to error on encountering back pressure. 458 | 459 | #### ReactiveJournalBackPressureBuffer 460 | 461 | In this program we set up `ReactiveJournal` to handle back pressure in the buffer mode 462 | but solving all the problems that we saw with the standard RxJava `BUFFER` mode. 463 | 464 | The FastProducer can be created with 465 | the `BackpressureStrategy.MISSING` because we don't expect that the producer will ever be 466 | slowed down by the consumer, which in this case is `RxRecorder`. 467 | 468 | The Consumer, rather than subscribing directly to the FastProducer, subscribes to 469 | `RxPlayer`. Note that `RxPlayer.play` returns an `Obserable` as there is no need for it to 470 | handle back pressure because bakc pressure has already been applied using `ReactiveJournal` as the 471 | buffer. 472 | 473 | Lets look at the problems `BackpressureStrategy.MISSING` and see how they are solved. 474 | 475 | * Even if the program crashes everything written to ReactiveJournal is safe. The events will be stored 476 | to disk and you can just restart the program and carry on consuming the queue at the point 477 | you crashed. If there is a OS/Machine level issue it is possible that a few messages might 478 | get lost that are waiting to be written to disk. 479 | If that is a problem you should make sure that replication is setup on your system. 480 | * There is no in-memory buffer so there is no need to run with extra heap memory and the program 481 | certainly won't run out memory because of `ReactiveJournal`. 482 | * When you call `RxPlayer.play` one of the the options is `using`. This allows you to pass in the 483 | object that will be used for every event. This means that no new objects will be allocated even 484 | if you have millions of items in your stream. (Of course if you want to hole a reference to the 485 | event you will need to clone). 486 | 487 | In addition to those benefits you will have the ususal benefits of using 'RxJornal' in that you 488 | will have a full record of the stream to use in testing and you will be able to use remote 489 | JVMs. 490 | 491 | #### ReactiveJournalBackPressureLatest 492 | 493 | As its name implies this demo program shows you how to handle back pressure using `ReactiveJournal` 494 | but rather than buffer you just want the latest item on the queue. 495 | 496 | All you have to do is set up the program exactly as we did in the previous example 497 | `ReactiveJournalBackPressureBuffer` but rather than the slow subscriber subscribing to the Observable 498 | that comes from `RxPlayer.play` we insert a `Flowable` inbetween. The `Flowable` is created 499 | with `BackpressureStrategy.LATEST`. 500 | 501 | See code snippet from the example below: 502 | 503 | ```java 504 | //1. Get the stream of events from the RxPlayer 505 | ConnectableObservable journalInput = reactiveJournal.createRxPlayer().play(options).publish(); 506 | 507 | //2. Create a Flowable with LATEST back pressure strategy from the ReactiveJournal stream 508 | Flowable flowable = journalInput.toFlowable(BackpressureStrategy.LATEST); 509 | 510 | //3. Record the output of the Flowable into the journal (note the different filter name) 511 | recorder.record(flowable, "consumed"); 512 | 513 | long startTime = System.currentTimeMillis(); 514 | 515 | //4. The slow consumer subscribes to the Flowable 516 | flowable.observeOn(Schedulers.io()).subscribe(onNextSlowConsumer::accept, 517 | e -> System.out.println("RxRecorder " + " " + e), 518 | () -> System.out.println("RxRecorder complete [" + (System.currentTimeMillis()-startTime) + "]") 519 | ); 520 | ``` 521 | 522 | You might have noticed that as well as the Slow Consumer subscribing to the Flowable to make 523 | sure it uses the LATEST strategy we also record the values we actaully consumer into ReactiveJournal. 524 | 525 | As with the plain RxJava implementation of LATEST (without ReactiveJournal) the Slow Consumer 526 | only sees the latest updates from the Fast Producer. However if you use RxRecorder (as in this 527 | example) you have: 528 | * A full record of all the events that were emitted by the Fast Producer. 529 | * A full record of all the events that were actaully consumed by the Slow Producer. 530 | 531 | Both these streams can be played back with `RxPlayer` by specifying the appropriate filter 532 | in the `PlayOptions` when calling `play`. 533 | 534 | This leads to being ablse to try the following... 535 | 536 | #### ReactiveJournalBackPressureTestingFasterConsumer 537 | 538 | In this example we experiment by replaying the event stream recorded in `ReactiveJournal` and 539 | observing the effects of lowering the latency of the SlowConsumer. 540 | 541 | We have a recording of the FastProducer created whilst running `ReactiveJournalBackPressureBuffer`. 542 | The SlowConsumer subscribes to this using a `Flowable` with `BackpressureStrategy.LATEST` 543 | as in the provious example. 544 | 545 | When we run with the SlowConsumer at a latency of 5ms we get this result: 546 | 547 | ```` 548 | Received [100] items. Published item[100] 549 | Received [200] items. Published item[391] 550 | Received [300] items. Published item[791] 551 | Received [400] items. Published item[1175] 552 | Received [500] items. Published item[1560] 553 | Received [600] items. Published item[1946] 554 | Received [700] items. Published item[2340] 555 | RxRecorder complete [3909ms] 556 | ```` 557 | 558 | The Slow Consumer has managed to consume about 700 events. 559 | 560 | If we reduce the latency of SlowConsumer to 3ms we get this result: 561 | 562 | ```` 563 | Received [100] items. Published item[100] 564 | Received [200] items. Published item[265] 565 | Received [300] items. Published item[491] 566 | Received [400] items. Published item[719] 567 | Received [500] items. Published item[958] 568 | Received [600] items. Published item[1192] 569 | Received [700] items. Published item[1428] 570 | Received [800] items. Published item[1664] 571 | Received [900] items. Published item[2046] 572 | Received [1000] items. Published item[2288] 573 | RxRecorder complete [3666ms] 574 | ```` 575 | 576 | The Slow Consumer has now managed to consume about 1000 events. 577 | 578 | Whilst this is a trivial example I'll let your imagination extend the scenarios 579 | to real world situations where this sort of ability to replay data against real 580 | load will be invaluable. 581 | 582 | ## Acknowlegments 583 | 584 | Special thanks to my friend and ex-collegue [Peter Lawrey](https://stackoverflow.com/users/57695/peter-lawrey) 585 | for inspiring me with his [Chronicle libraries](https://github.com/OpenHFT) which underpin ReactiveJournal. 586 | 587 | To those behind [RxJava](https://github.com/ReactiveX/RxJava) in particular to 588 | [Tomasz Nurkiewicz](http://www.nurkiewicz.com) for his talks and book which 589 | opened my eyes to RxJava. 590 | 591 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | october-technologies 8 | rxrecorder 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 13 | net.openhft 14 | chronicle-queue 15 | 4.5.27 16 | 17 | 18 | io.reactivex.rxjava2 19 | rxjava 20 | 2.1.0 21 | 22 | 23 | junit 24 | junit 25 | 4.12 26 | 27 | 28 | org.slf4j 29 | slf4j-simple 30 | 1.7.14 31 | 32 | 33 | 34 | 35 | 36 | 37 | maven-compiler-plugin 38 | 3.1 39 | 40 | 1.8 41 | 1.8 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/fastproducerslowconsumer/FastProducerSlowConsumer.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.fastproducerslowconsumer; 2 | 3 | import io.reactivex.BackpressureStrategy; 4 | import io.reactivex.Flowable; 5 | import org.reactivejournal.util.DSUtil; 6 | 7 | import java.util.concurrent.atomic.AtomicInteger; 8 | import java.util.concurrent.atomic.AtomicLong; 9 | import java.util.function.Consumer; 10 | 11 | /** 12 | * Created by daniel on 24/05/17. 13 | */ 14 | public class FastProducerSlowConsumer { 15 | static Flowable createFastProducer(BackpressureStrategy backpressureStrategy, int stopAfter){ 16 | return Flowable.create(emitter -> { 17 | AtomicLong publishedCount = new AtomicLong(0); 18 | while (true) { 19 | publishedCount.incrementAndGet(); 20 | if (publishedCount.get() == stopAfter) { 21 | emitter.onComplete(); 22 | break; 23 | } 24 | DSUtil.sleep(1); 25 | emitter.onNext(publishedCount.get()); 26 | } 27 | }, backpressureStrategy); 28 | } 29 | 30 | static Consumer createOnNextSlowConsumer(int delayMS){ 31 | AtomicInteger countReceived = new AtomicInteger(0); 32 | 33 | return nextItem -> { 34 | countReceived.incrementAndGet(); 35 | if (countReceived.get() % 100 == 0) { 36 | System.out.println("Received [" + countReceived.get() + "] items. Published item[" + nextItem + "]"); 37 | } 38 | 39 | DSUtil.sleep(delayMS); 40 | }; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/fastproducerslowconsumer/RxJavaBackPressure.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.fastproducerslowconsumer; 2 | 3 | import io.reactivex.BackpressureStrategy; 4 | import io.reactivex.Flowable; 5 | import io.reactivex.schedulers.Schedulers; 6 | import org.reactivejournal.util.DSUtil; 7 | 8 | import java.util.function.Consumer; 9 | 10 | /** 11 | * Sample program to show how RxJava deals with back pressure 12 | */ 13 | public class RxJavaBackPressure { 14 | 15 | public static void main(String[] args) { 16 | run(BackpressureStrategy.BUFFER); 17 | run(BackpressureStrategy.LATEST); 18 | run(BackpressureStrategy.DROP); 19 | run(BackpressureStrategy.MISSING); 20 | run(BackpressureStrategy.ERROR); 21 | } 22 | 23 | private static void run(BackpressureStrategy backpressureStrategy) { 24 | System.out.println("-- RUNNING WITH [" + backpressureStrategy + "] --"); 25 | Flowable fastProducer = FastProducerSlowConsumer.createFastProducer(backpressureStrategy, 1000); 26 | 27 | Consumer onNextSlowConsumer = FastProducerSlowConsumer.createOnNextSlowConsumer(5); 28 | fastProducer.observeOn(Schedulers.io()).subscribe(onNextSlowConsumer::accept, 29 | e -> System.out.println(backpressureStrategy + " " + e), 30 | () -> System.out.println(backpressureStrategy + " complete") 31 | ); 32 | if(backpressureStrategy== BackpressureStrategy.BUFFER)DSUtil.sleep(5000); 33 | DSUtil.sleep(1000); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/fastproducerslowconsumer/RxJournalBackPressureBuffer.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.fastproducerslowconsumer; 2 | 3 | import io.reactivex.BackpressureStrategy; 4 | import io.reactivex.Flowable; 5 | import org.reactivejournal.impl.PlayOptions; 6 | import org.reactivejournal.impl.ReactiveJournal; 7 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 8 | 9 | import java.io.IOException; 10 | import java.util.function.Consumer; 11 | 12 | /** 13 | * Program to demonstrate how ReactiveJournal handles back pressure. 14 | */ 15 | public class RxJournalBackPressureBuffer { 16 | 17 | public static void main(String[] args) throws IOException { 18 | 19 | ReactiveJournal reactiveJournal = new ReactiveJournal("/tmp/fastproducer"); 20 | reactiveJournal.clearCache(); 21 | Flowable fastProducer = FastProducerSlowConsumer.createFastProducer(BackpressureStrategy.MISSING, 500); 22 | 23 | reactiveJournal.createReactiveRecorder().recordAsync(fastProducer,"input"); 24 | PlayOptions options = new PlayOptions().filter("input").replayRate(PlayOptions.ReplayRate.FAST); 25 | Flowable journalInput = new RxJavaPlayer(reactiveJournal).play(options); 26 | 27 | Consumer onNextSlowConsumer = FastProducerSlowConsumer.createOnNextSlowConsumer(10); 28 | 29 | long startTime = System.currentTimeMillis(); 30 | journalInput.subscribe(onNextSlowConsumer::accept, 31 | e -> System.out.println("ReactiveRecorder " + " " + e), 32 | () -> System.out.println("ReactiveRecorder complete [" + (System.currentTimeMillis()-startTime) + "]") 33 | ); 34 | 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/fastproducerslowconsumer/RxJournalBackPressureLatest.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.fastproducerslowconsumer; 2 | 3 | import io.reactivex.BackpressureStrategy; 4 | import io.reactivex.Flowable; 5 | import io.reactivex.flowables.ConnectableFlowable; 6 | import io.reactivex.schedulers.Schedulers; 7 | import org.reactivejournal.impl.PlayOptions; 8 | import org.reactivejournal.impl.ReactiveJournal; 9 | import org.reactivejournal.impl.ReactiveRecorder; 10 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 11 | import org.reactivejournal.util.DSUtil; 12 | 13 | import java.io.IOException; 14 | import java.util.function.Consumer; 15 | 16 | /** 17 | * Program to demonstrate how ReactiveJournal handles back pressure if you want LATEST rather than Buffer. 18 | */ 19 | public class RxJournalBackPressureLatest { 20 | 21 | public static void main(String[] args) throws IOException { 22 | 23 | ReactiveJournal reactiveJournal = new ReactiveJournal("/tmp/fastproducer"); 24 | reactiveJournal.clearCache(); 25 | Flowable fastProducer = FastProducerSlowConsumer.createFastProducer(BackpressureStrategy.MISSING, 2500); 26 | 27 | ReactiveRecorder recorder = reactiveJournal.createReactiveRecorder(); 28 | recorder.recordAsync(fastProducer,"input"); 29 | //Set the replay strategy to ReplayRate.FAST as e want to process the event as soon as it is 30 | //received from the publisher. 31 | PlayOptions options = new PlayOptions().filter("input").replayRate(PlayOptions.ReplayRate.FAST); 32 | ConnectableFlowable journalInput = new RxJavaPlayer(reactiveJournal).play(options).publish(); 33 | 34 | Consumer onNextSlowConsumer = FastProducerSlowConsumer.createOnNextSlowConsumer(10); 35 | 36 | recorder.record(journalInput, "consumed"); 37 | 38 | long startTime = System.currentTimeMillis(); 39 | journalInput.observeOn(Schedulers.io()).subscribe(onNextSlowConsumer::accept, 40 | e -> System.out.println("ReactiveRecorder " + " " + e), 41 | () -> System.out.println("ReactiveRecorder complete [" + (System.currentTimeMillis()-startTime) + "]") 42 | ); 43 | 44 | journalInput.connect(); 45 | 46 | DSUtil.sleep(3000); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/fastproducerslowconsumer/RxJournalBackPressureTestingFasterConsumer.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.fastproducerslowconsumer; 2 | 3 | import io.reactivex.flowables.ConnectableFlowable; 4 | import io.reactivex.schedulers.Schedulers; 5 | import org.reactivejournal.impl.PlayOptions; 6 | import org.reactivejournal.impl.ReactiveJournal; 7 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 8 | import org.reactivejournal.util.DSUtil; 9 | 10 | import java.io.IOException; 11 | import java.util.function.Consumer; 12 | 13 | import static org.reactivejournal.impl.PlayOptions.ReplayRate; 14 | 15 | /** 16 | * Program to demonstrate how you can replay from the journal to test whether you have improved the 17 | * speed of your program. 18 | * 19 | * Using the recording created by RxJournalBackPressureLatest we see how many more items 20 | * we can consume if we speed up our consumer. 21 | */ 22 | public class RxJournalBackPressureTestingFasterConsumer { 23 | public static void main(String[] args) throws IOException { 24 | ReactiveJournal reactiveJournal = new ReactiveJournal("src/main/java/org/reactivejournal/examples/fastproducerslowconsumer/resources/"); 25 | 26 | //Get the input from the recorder note that we have to set the replayRate to ACTUAL_TIME 27 | //to replicate the conditions in the 'real world'. 28 | PlayOptions options = new PlayOptions().filter("input").replayRate(ReplayRate.ACTUAL_TIME); 29 | ConnectableFlowable journalInput = new RxJavaPlayer(reactiveJournal).play(options).publish(); 30 | 31 | //Reduce the latency of the consumer to 5ms - try reducing or increasing to study the effects. 32 | Consumer onNextSlowConsumer = FastProducerSlowConsumer.createOnNextSlowConsumer(3); 33 | 34 | long startTime = System.currentTimeMillis(); 35 | journalInput.observeOn(Schedulers.io()).subscribe(onNextSlowConsumer::accept, 36 | e -> System.out.println("ReactiveRecorder " + " " + e), 37 | () -> System.out.println("ReactiveRecorder complete [" + (System.currentTimeMillis()-startTime) + "ms]") 38 | ); 39 | 40 | journalInput.connect(); 41 | 42 | DSUtil.sleep(1000); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/fastproducerslowconsumer/resources/20170525.cq4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danielshaya/reactivejournal/cc441731729cba62457481d94489ca97b8966545/src/main/java/org/reactivejournal/examples/fastproducerslowconsumer/resources/20170525.cq4 -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/helloworld/BytesToWordsProcessor.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.helloworld; 2 | 3 | import io.reactivex.Flowable; 4 | import io.reactivex.processors.PublishProcessor; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | 9 | /** 10 | * A simple BytesToWordsProcessor program. Receives a stream of Bytes and converts to 11 | * a stream of words. (This is intended for demonstration purposes only.) 12 | */ 13 | public class BytesToWordsProcessor { 14 | private static final Logger LOG = LoggerFactory.getLogger(BytesToWordsProcessor.class.getName()); 15 | public Flowable process(Flowable observableInput){ 16 | 17 | PublishProcessor publishProcessor = PublishProcessor.create(); 18 | 19 | StringBuilder sb = new StringBuilder(); 20 | observableInput.subscribe(b->{ 21 | if(b==32){ //send out a new word on a space 22 | publishProcessor.onNext(sb.toString()); 23 | sb.setLength(0); 24 | }else{ 25 | sb.append((char)b.byteValue()); 26 | } 27 | }, 28 | e->LOG.error("Error in BytesToWordsProcessor [{}]", e), 29 | publishProcessor::onComplete 30 | ); 31 | 32 | return publishProcessor; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/helloworld/HelloWorld.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.helloworld; 2 | 3 | import org.reactivejournal.impl.PlayOptions; 4 | import org.reactivejournal.impl.ReactiveJournal; 5 | import org.reactivejournal.impl.ReactiveRecorder; 6 | import org.reactivestreams.Publisher; 7 | import org.reactivestreams.Subscriber; 8 | import org.reactivestreams.Subscription; 9 | 10 | import java.io.File; 11 | import java.io.IOException; 12 | 13 | /** 14 | * Simple Demo Program 15 | */ 16 | public class HelloWorld { 17 | private static final String tmpDir = System.getProperty("java.io.tmpdir"); 18 | 19 | public static void main(String[] args) throws IOException { 20 | //1. Create the reactiveRecorder and delete any previous content by clearing the cache 21 | ReactiveJournal reactiveJournal = new ReactiveJournal(tmpDir + File.separator + "HW"); 22 | reactiveJournal.clearCache(); 23 | 24 | //2. Create a HelloWorld Publisher 25 | Publisher helloWorldFlowable = subscriber -> { 26 | subscriber.onNext("Hello World!"); 27 | subscriber.onComplete(); 28 | }; 29 | 30 | //3. Pass the Publisher into the reactiveRecorder which will subscribe to it and record all events. 31 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 32 | reactiveRecorder.record(helloWorldFlowable, ""); 33 | 34 | //4. Subscribe to ReactiveJournal and print out results 35 | Publisher recordedObservable = reactiveJournal.createReactivePlayer().play(new PlayOptions()); 36 | 37 | recordedObservable.subscribe(new Subscriber() { 38 | @Override 39 | public void onSubscribe(Subscription subscription) { 40 | subscription.request(Long.MAX_VALUE); 41 | } 42 | 43 | @Override 44 | public void onNext(Object o) { 45 | System.out.println(o); 46 | } 47 | 48 | @Override 49 | public void onError(Throwable throwable) { 50 | System.err.println(throwable); 51 | } 52 | 53 | @Override 54 | public void onComplete() { 55 | System.out.println("Hello World Complete"); 56 | } 57 | }); 58 | 59 | //5. Sometimes useful to see the recording written to a file 60 | reactiveJournal.writeToFile(tmpDir + File.separator + "/hw.txt",true); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/helloworld/HelloWorldApp_JounalAsObserver.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.helloworld; 2 | 3 | import io.reactivex.Flowable; 4 | import io.reactivex.flowables.ConnectableFlowable; 5 | import org.reactivejournal.impl.ReactiveJournal; 6 | import org.reactivejournal.impl.ReactiveRecorder; 7 | import org.reactivejournal.util.DSUtil; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.io.IOException; 12 | 13 | /** 14 | * An example class that demonstrates how use ReactiveRecorder as a second subscriber 15 | * to the flowableInput. i.e. The BytesToWordsProcessor subscribes to the flowableInput 16 | * as does the ReactiveRecorder. 17 | * Note that everything happens on the same thread. 18 | */ 19 | public class HelloWorldApp_JounalAsObserver { 20 | private static final Logger LOG = LoggerFactory.getLogger(HelloWorldApp_JounalAsObserver.class.getName()); 21 | 22 | public final static String FILE_NAME = System.getProperty("filename", "/tmp/Demo"); 23 | public final static int INTERVAL_MS = 100; 24 | public final static String INPUT_FILTER = "input"; 25 | public final static String OUTPUT_FILTER = "output"; 26 | 27 | public static void main(String[] args) throws IOException { 28 | ConnectableFlowable flowableInput = 29 | Flowable.fromArray(new Byte[]{72,101,108,108,111,32,87,111,114,108,100,32}).map( 30 | i->{ 31 | DSUtil.sleep(INTERVAL_MS); 32 | return i; 33 | }).publish(); 34 | 35 | //Create the reactiveRecorder and delete any previous content by clearing the cache 36 | ReactiveJournal reactiveJournal = new ReactiveJournal(FILE_NAME); 37 | reactiveJournal.clearCache(); 38 | 39 | //Pass the input stream into the reactiveRecorder which will subscribe to it and record all events. 40 | //The subscription will not be activated until 'connect' is called on the input stream. 41 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 42 | reactiveRecorder.record(flowableInput, INPUT_FILTER); 43 | 44 | BytesToWordsProcessor bytesToWords = new BytesToWordsProcessor(); 45 | //Pass the input Byte stream into the BytesToWordsProcessor class which subscribes to the stream and returns 46 | //a stream of words. 47 | //The subscription will not be activated until 'connect' is called on the input stream. 48 | Flowable flowableOutput = bytesToWords.process(flowableInput); 49 | 50 | //Pass the output stream (of words) into the reactiveRecorder which will subscribe to it and record all events. 51 | flowableOutput.subscribe(LOG::info); 52 | reactiveRecorder.record(flowableOutput, OUTPUT_FILTER); 53 | 54 | //Activate the subscriptions 55 | flowableInput.connect(); 56 | 57 | reactiveJournal.writeToFile("/tmp/Demo/demo.txt",true); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/helloworld/HelloWorldApp_JournalPlayThrough.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.helloworld; 2 | 3 | import io.reactivex.Flowable; 4 | import io.reactivex.flowables.ConnectableFlowable; 5 | import org.reactivejournal.impl.PlayOptions; 6 | import org.reactivejournal.impl.ReactiveJournal; 7 | import org.reactivejournal.impl.ReactiveRecorder; 8 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 9 | import org.reactivejournal.util.DSUtil; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.io.IOException; 14 | 15 | /** 16 | * Class to run BytesToWordsProcessor 17 | */ 18 | public class HelloWorldApp_JournalPlayThrough { 19 | private static final Logger LOG = LoggerFactory.getLogger(HelloWorldApp_JournalPlayThrough.class.getName()); 20 | 21 | public final static String FILE_NAME = System.getProperty("filename", "/tmp/Demo"); 22 | public final static int INTERVAL_MS = 10; 23 | public final static String INPUT_FILTER = "input"; 24 | public final static String OUTPUT_FILTER = "output"; 25 | public static final Flowable observableInput = 26 | Flowable.fromArray(new Byte[]{72,101,108,108,111,32,87,111,114,108,100,32}).map( 27 | i->{ 28 | DSUtil.sleep(INTERVAL_MS); 29 | return i; 30 | }); 31 | 32 | public static void main(String[] args) throws IOException { 33 | 34 | //Create the reactiveRecorder and delete any previous content by clearing the cache 35 | ReactiveJournal reactiveJournal = new ReactiveJournal(FILE_NAME); 36 | reactiveJournal.clearCache(); 37 | 38 | //Pass the input stream into the reactiveRecorder which will subscribe to it and record all events. 39 | //The subscription will not be activated on a new thread which will allow this program to continue. 40 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 41 | reactiveRecorder.recordAsync(observableInput, INPUT_FILTER); 42 | 43 | BytesToWordsProcessor bytesToWords = new BytesToWordsProcessor(); 44 | 45 | //Retrieve a stream of 46 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 47 | PlayOptions options = new PlayOptions().filter(INPUT_FILTER).playFromNow(true); 48 | ConnectableFlowable recordedObservable = rxPlayer.play(options).publish(); 49 | //Pass the input Byte stream into the BytesToWordsProcessor class which subscribes to the stream and returns 50 | //a stream of words. 51 | Flowable flowableOutput = bytesToWords.process(recordedObservable); 52 | 53 | //Pass the output stream (of words) into the reactiveRecorder which will subscribe to it and record all events. 54 | reactiveRecorder.record(flowableOutput, OUTPUT_FILTER); 55 | flowableOutput.subscribe(s -> LOG.info("HelloWorldHot->" + s), 56 | throwable -> LOG.error("", throwable), 57 | ()->LOG.info("HelloWorldHot Complete")); 58 | //Only start the recording now because we want to make sure that the BytesToWordsProcessor and the reactiveRecorder 59 | //are both setup up to receive subscriptions. 60 | recordedObservable.connect(); 61 | //Sometimes useful to see the recording written to a file 62 | reactiveJournal.writeToFile("/tmp/Demo/demo.txt",true); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/helloworld/HelloWorldRemote.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.helloworld; 2 | 3 | import io.reactivex.Flowable; 4 | import io.reactivex.flowables.ConnectableFlowable; 5 | import org.reactivejournal.impl.PlayOptions; 6 | import org.reactivejournal.impl.ReactiveJournal; 7 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.io.IOException; 12 | 13 | /** 14 | * 15 | */ 16 | public class HelloWorldRemote { 17 | private static final Logger LOG = LoggerFactory.getLogger(HelloWorldRemote.class.getName()); 18 | 19 | public static void main(String... args) throws IOException, InterruptedException { 20 | //Create the rxRecorder but don't delete the cache that has been created. 21 | ReactiveJournal reactiveJournal = new ReactiveJournal(HelloWorldApp_JounalAsObserver.FILE_NAME); 22 | //Get the input from the remote process 23 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 24 | PlayOptions options = new PlayOptions().filter(HelloWorldApp_JounalAsObserver.INPUT_FILTER) 25 | .playFromNow(true).replayRate(PlayOptions.ReplayRate.FAST); 26 | ConnectableFlowable remoteInput = rxPlayer.play(options).publish(); 27 | 28 | BytesToWordsProcessor bytesToWords = new BytesToWordsProcessor(); 29 | Flowable flowableOutput = bytesToWords.process(remoteInput); 30 | 31 | 32 | flowableOutput.subscribe( 33 | s->LOG.info("Remote input [{}]", s), 34 | e-> LOG.error("Problem in remote [{}]", e), 35 | ()->{ 36 | LOG.info("Remote input ended"); 37 | System.exit(0); 38 | }); 39 | 40 | remoteInput.connect(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/helloworld/HelloWorldTest.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.helloworld; 2 | 3 | import io.reactivex.Flowable; 4 | import io.reactivex.flowables.ConnectableFlowable; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | import org.reactivejournal.impl.ReactiveValidator; 8 | import org.reactivestreams.Subscriber; 9 | import org.reactivestreams.Subscription; 10 | import org.reactivejournal.impl.PlayOptions; 11 | import org.reactivejournal.impl.ReactiveJournal; 12 | import org.reactivejournal.impl.ValidationResult; 13 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | import java.io.IOException; 18 | import java.util.concurrent.CountDownLatch; 19 | import java.util.concurrent.TimeUnit; 20 | 21 | /** 22 | * A demo example Junit test class to test BytesToWordsProcessor. 23 | * Make sure you have run the HelloWorldApp_JounalAsObserver first to generate the journal. 24 | */ 25 | public class HelloWorldTest { 26 | private static final Logger LOG = LoggerFactory.getLogger(HelloWorldTest.class.getName()); 27 | 28 | @Test 29 | public void testHelloWorld() throws IOException, InterruptedException { 30 | //Create the rxRecorder but don't delete the cache that has been created. 31 | ReactiveJournal reactiveJournal = new ReactiveJournal(HelloWorldApp_JounalAsObserver.FILE_NAME); 32 | //reactiveJournal.writeToFile("/tmp/Demo/demo.txt", true); 33 | 34 | //Get the input from the recorder 35 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 36 | //In this case we can play the data stream in FAST mode. 37 | PlayOptions options= new PlayOptions().filter(HelloWorldApp_JounalAsObserver.INPUT_FILTER) 38 | .replayRate(PlayOptions.ReplayRate.FAST).sameThread(true); 39 | //Use a ConnectableObservable as we only want to kick off the stream when all 40 | //connections have been wired together. 41 | ConnectableFlowable observableInput = rxPlayer.play(options).publish(); 42 | 43 | BytesToWordsProcessor bytesToWords = new BytesToWordsProcessor(); 44 | Flowable flowableOutput = bytesToWords.process(observableInput); 45 | 46 | CountDownLatch latch = new CountDownLatch(1); 47 | //Send the output stream to the recorder to be validated against the recorded output 48 | ReactiveValidator reactiveValidator = reactiveJournal.createReactiveValidator(); 49 | reactiveValidator.validate(HelloWorldApp_JounalAsObserver.FILE_NAME + "/.reactiveJournal", 50 | flowableOutput, HelloWorldApp_JounalAsObserver.OUTPUT_FILTER, new Subscriber() { 51 | @Override 52 | public void onSubscribe(Subscription subscription) { 53 | subscription.request(Long.MAX_VALUE); 54 | } 55 | 56 | @Override 57 | public void onNext(Object o) { 58 | LOG.info(o.toString()); 59 | } 60 | 61 | @Override 62 | public void onError(Throwable throwable) { 63 | LOG.error("Problem in process test [{}]", throwable); 64 | } 65 | 66 | @Override 67 | public void onComplete() { 68 | LOG.info("Summary[" + reactiveValidator.getValidationResult().summaryResult() 69 | + "] items compared[" + reactiveValidator.getValidationResult().summaryItemsCompared() 70 | + "] items valid[" + reactiveValidator.getValidationResult().summaryItemsValid() +"]"); 71 | latch.countDown(); 72 | } 73 | }); 74 | 75 | observableInput.connect(); 76 | boolean completedWithoutTimeout = latch.await(200, TimeUnit.SECONDS); 77 | Assert.assertEquals(ValidationResult.Result.OK, reactiveValidator.getValidationResult().getResult()); 78 | Assert.assertTrue(completedWithoutTimeout); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/examples/helloworld/HelloWorldUsingRxJava.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.examples.helloworld; 2 | 3 | import io.reactivex.Flowable; 4 | import org.reactivejournal.impl.PlayOptions; 5 | import org.reactivejournal.impl.ReactiveJournal; 6 | import org.reactivejournal.impl.ReactiveRecorder; 7 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 8 | 9 | import java.io.File; 10 | import java.io.IOException; 11 | 12 | /** 13 | * Simple Demo Program using ReactiveJournal integrated with RXJava 14 | */ 15 | public class HelloWorldUsingRxJava { 16 | private static final String tmpDir = System.getProperty("java.io.tmpdir"); 17 | 18 | public static void main(String[] args) throws IOException { 19 | //1. Create the reactiveRecorder and delete any previous content by clearing the cache 20 | ReactiveJournal reactiveJournal = new ReactiveJournal(tmpDir + File.separator + "HW"); 21 | reactiveJournal.clearCache(); 22 | 23 | //2. Create an RxJava Hello World Flowable 24 | Flowable helloWorldFlowable = Flowable.just("Hello World!!"); 25 | //3. Pass the flowable into the reactiveRecorder which will subscribe to it and record all events. 26 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 27 | reactiveRecorder.record(helloWorldFlowable, ""); 28 | 29 | //4. Use the utility class RxJavaPlayer to subscribe to the journal 30 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 31 | Flowable recordedObservable = rxPlayer.play(new PlayOptions()); 32 | 33 | //5. Print out what we get as a callback to onNext(). 34 | recordedObservable.subscribe(System.out::println); 35 | 36 | //6. Sometimes useful to see the recording written to a file 37 | reactiveJournal.writeToFile(tmpDir + File.separator + "/hw.txt",true); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/impl/DataItemProcessor.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import net.openhft.chronicle.wire.ValueIn; 4 | 5 | /** 6 | * Created by daniel on 18/05/17. 7 | */ 8 | public class DataItemProcessor { 9 | 10 | private byte status; 11 | private long messageCount; 12 | private long time; 13 | private String storedFilter; 14 | private Object valueFromQueue; 15 | 16 | public void process(ValueIn in, Object using){ 17 | status = in.int8(); 18 | messageCount = in.int64(); 19 | time = in.int64(); 20 | storedFilter = in.text(); 21 | 22 | if(status == ReactiveStatus.ERROR){ 23 | valueFromQueue = in.throwable(false); 24 | }else { 25 | if(using== null) { 26 | valueFromQueue = in.object(); 27 | }else{ 28 | valueFromQueue = in.object(using, using.getClass()); 29 | } 30 | } 31 | } 32 | 33 | public byte getStatus() { 34 | return status; 35 | } 36 | 37 | public long getMessageCount() { 38 | return messageCount; 39 | } 40 | 41 | public long getTime() { 42 | return time; 43 | } 44 | 45 | public String getFilter() { 46 | return storedFilter; 47 | } 48 | 49 | public Object getObject() { 50 | return valueFromQueue; 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | return "DataItemProcessor{" + 56 | "status=" + status + 57 | ", messageCount=" + messageCount + 58 | ", time=" + time + 59 | ", storedFilter='" + storedFilter + '\'' + 60 | ", valueFromQueue=" + valueFromQueue + 61 | '}'; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/impl/EndOfStream.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import net.openhft.chronicle.wire.Marshallable; 4 | 5 | /** 6 | * Created by daniel on 25/04/17. 7 | */ 8 | public class EndOfStream implements Marshallable { 9 | @Override 10 | public String toString() { 11 | return "EndOfStream{}"; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/impl/PlayOptions.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | /** 4 | * A container class to encapsulate all the configurations settings that can be passed 5 | * into the play() method of {@link ReactivePlayer} 6 | */ 7 | public class PlayOptions { 8 | private String filter = ""; 9 | private ReplayRate replayRate = ReplayRate.FAST; 10 | private PauseStrategy pauseStrategy = PauseStrategy.YIELD; 11 | private Object using = null; 12 | private boolean playFromNow = false; 13 | private long playFromTime = Long.MIN_VALUE; 14 | private long playUntilTime = Long.MAX_VALUE; 15 | private long playFromSeqNo = Long.MIN_VALUE; 16 | private long playUntilSeqNo = Long.MAX_VALUE; 17 | private boolean completeAtEndOfFile = false; 18 | private boolean sameThread = false; 19 | 20 | String filter() { 21 | return filter; 22 | } 23 | 24 | /** 25 | * Allows the user to play back the events in the journal which weretagged with this filter. 26 | * Typical examples would be recording input events 27 | * to a process with 'input' and results of the processor with 'output'. 28 | * @param filter The events in the journal saved with the tag 29 | * @return PlayOptions for use in the Builder pattern 30 | */ 31 | public PlayOptions filter(String filter) { 32 | this.filter = filter; 33 | return this; 34 | } 35 | 36 | ReplayRate replayRate() { 37 | return replayRate; 38 | } 39 | 40 | /** 41 | * When play() is called, RxPlayer can either play the events back in ACTUAL_TIME 42 | * respecting the gaps between the originally recoded events, or FAST, playing 43 | * the evnts back as fast as it can. Typically for testing you would use ACTUAL_TIME 44 | * whereas for remote and back pressure usages you would use FAST. 45 | * 46 | * @param replayRateStrategy strategy determining the replay rate. 47 | * @return PlayOptions for use in the Builder pattern 48 | */ 49 | public PlayOptions replayRate(ReplayRate replayRateStrategy) { 50 | this.replayRate = replayRateStrategy; 51 | return this; 52 | } 53 | 54 | PauseStrategy pauseStrategy() { 55 | return pauseStrategy; 56 | } 57 | 58 | /** 59 | * When polling for events the pause strategy is set by {@link PauseStrategy}. 60 | * This will default to PauseStrategy.YIELD which calls Thread.yield() 61 | * between each unsuccessful attempt to call for the next event. This is 62 | * suitable for most applications.

63 | * In low latency senitive applications however you might want to set the 64 | * {@link PauseStrategy} to PauseStrategy.SPIN which will poll continuously 65 | * taking up a full CPU. This will reduce latency to get an event into the 66 | * data stream but you need to make sure you have the available compute 67 | * power. 68 | * 69 | * @param pauseStrategy The PauseStrategy to be applied 70 | * @return PlayOptions for use in the Builder pattern 71 | */ 72 | public PlayOptions pauseStrategy(PauseStrategy pauseStrategy) { 73 | this.pauseStrategy = pauseStrategy; 74 | return this; 75 | } 76 | 77 | Object using() { 78 | return using; 79 | } 80 | 81 | /** 82 | * Where keeping GC to a minimum is a requirement the program should not 83 | * create a new object for every data event. Instead we can just repopulate 84 | * the same object each time. There is an assumption that the getters/setters 85 | * are all available. With this method you are able to pass in a protoype object 86 | * which will be repopulated with the data for each event. 87 | *

88 | * It is important to remember that if you want to keep a reference to the object 89 | * that comes off the stream you will need to clone the object. 90 | * 91 | * @param using The prototype object to be used to populate data from the stream 92 | * @return PlayOptions for use in the Builder pattern 93 | */ 94 | public PlayOptions using(Object using) { 95 | this.using = using; 96 | return this; 97 | } 98 | 99 | /** 100 | * This setting is used when you only want to listen to events put on the stream 101 | * from now and you are not interested in replaying from the start of the recording. 102 | * This might be true, for example, if you have a remote connection to a hot event 103 | * source. 104 | * 105 | * @param playFromNow Defaulted to false 106 | * @return PlayOptions for use in the Builder pattern 107 | */ 108 | public PlayOptions playFromNow(boolean playFromNow) { 109 | this.playFromNow = playFromNow; 110 | return this; 111 | } 112 | 113 | long playFromTime() { 114 | return playFromTime; 115 | } 116 | 117 | /** 118 | * Play back the recording from this time, Useful if you only want to play a subset of the recording. 119 | * @param playFromTime time in millis from 1970 120 | * @return PlayOptions for use in the Builder pattern 121 | */ 122 | public PlayOptions playFromTime(long playFromTime) { 123 | this.playFromTime = playFromTime; 124 | return this; 125 | } 126 | 127 | long playUntilTime() { 128 | return playUntilTime; 129 | } 130 | 131 | /** 132 | * Play back until this time. Useful if you only want to play a subset of the recording. 133 | * @param playUntilTime time in millis from 1970 134 | * @return PlayOptions for use in the Builder pattern 135 | */ 136 | public PlayOptions playUntilTime(long playUntilTime) { 137 | this.playUntilTime = playUntilTime; 138 | return this; 139 | } 140 | 141 | 142 | long playFromSeqNo() { 143 | return playFromSeqNo; 144 | } 145 | 146 | /** 147 | * Play back the recording from this seqNo, Useful if you only want to play a subset of the recording. 148 | * @param playFromSeqNo sequence number to start from 149 | * @return PlayOptions for use in the Builder pattern 150 | */ 151 | public PlayOptions playFromSeqNo(long playFromSeqNo) { 152 | this.playFromSeqNo = playFromSeqNo; 153 | return this; 154 | } 155 | 156 | long playUntilSeqNo() { 157 | return playUntilSeqNo; 158 | } 159 | 160 | /** 161 | * Play back until this sequence number. Useful if you only want to play a subset of the recording. 162 | * @param playUntilSeqNo play until this sequence number 163 | * @return PlayOptions for use in the Builder pattern 164 | */ 165 | public PlayOptions playUntilSeqNo(long playUntilSeqNo) { 166 | this.playUntilSeqNo = playUntilSeqNo; 167 | return this; 168 | } 169 | 170 | boolean completeAtEndOfFile() { 171 | return completeAtEndOfFile; 172 | } 173 | 174 | /** 175 | * Used when you want to stop as soon as there are no more items. For example 176 | * in a unit test. This is only useful where there was no completion item. 177 | * 178 | * @param waitForMoreItems Defaulted to true 179 | * @return PlayOptions for use in the Builder pattern 180 | */ 181 | public PlayOptions completeAtEndOfFile(boolean waitForMoreItems) { 182 | this.completeAtEndOfFile = waitForMoreItems; 183 | return this; 184 | } 185 | 186 | boolean sameThread() { 187 | return sameThread; 188 | } 189 | 190 | /** 191 | * By default the subscriber will be called back on a different thread. 192 | * 193 | * @param sameThread set the callback on the same thread 194 | * @return PlayOptions for use in the Builder pattern 195 | */ 196 | public PlayOptions sameThread(boolean sameThread) { 197 | this.sameThread = sameThread; 198 | return this; 199 | } 200 | 201 | void validate() throws IllegalArgumentException { 202 | if(playFromTime != Long.MIN_VALUE && playFromNow){ 203 | throw new IllegalArgumentException("Illegal combination: PlayFromTime and PlayFromNow" + 204 | "not allowed to be set at the same time"); 205 | } 206 | 207 | if(playFromTime != Long.MIN_VALUE && playFromSeqNo != Long.MIN_VALUE){ 208 | throw new IllegalArgumentException("Illegal combination: PlayFromTime and PlayFromSeqNo" + 209 | "not allowed to be set at the same time"); 210 | } 211 | 212 | if(playFromSeqNo != Long.MIN_VALUE && playFromNow){ 213 | throw new IllegalArgumentException("Illegal combination: PlayFromSeqNo and PlayFromNow" + 214 | "not allowed to be set at the same time"); 215 | } 216 | 217 | if(playUntilTime != Long.MAX_VALUE && playUntilSeqNo != Long.MAX_VALUE){ 218 | throw new IllegalArgumentException("Illegal combination: PlayUntiTime and PlayUntilSeqNo" + 219 | "not allowed to be set at the same time"); 220 | } 221 | 222 | if(playFromNow){ 223 | playFromTime = System.currentTimeMillis(); 224 | } 225 | } 226 | 227 | public enum ReplayRate {ACTUAL_TIME, FAST} 228 | 229 | public enum PauseStrategy {SPIN, YIELD} 230 | } 231 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/impl/ReactiveJournal.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import net.openhft.chronicle.queue.ChronicleQueue; 4 | import net.openhft.chronicle.queue.ExcerptTailer; 5 | import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder; 6 | import net.openhft.chronicle.wire.ValueIn; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.io.File; 11 | import java.io.FileWriter; 12 | import java.io.IOException; 13 | import java.nio.file.DirectoryNotEmptyException; 14 | import java.nio.file.Files; 15 | import java.nio.file.Path; 16 | import java.nio.file.Paths; 17 | import java.time.Instant; 18 | import java.time.LocalDateTime; 19 | import java.time.ZoneId; 20 | import java.util.concurrent.atomic.AtomicLong; 21 | 22 | /** 23 | * ReactiveJournal 24 | */ 25 | public class ReactiveJournal { 26 | private static final Logger LOG = LoggerFactory.getLogger(ReactiveJournal.class.getName()); 27 | 28 | /** 29 | * The default name for the data directory. 30 | */ 31 | public static final String REACTIVE_JOURNAL_DIRNAME = ".reactiveJournal"; 32 | 33 | private String dir; 34 | private final AtomicLong messageCounter = new AtomicLong(0); 35 | 36 | /** 37 | * Create an {@link ReactiveJournal} that will store or read its data from the {@value REACTIVE_JOURNAL_DIRNAME} directory 38 | * inside the given {@literal baseDir}. Use {@link #getDir()} to obtain the full path of said directory, and 39 | * {@link #clearCache()} to clean that directory of ReactiveJournal-specific files. 40 | * 41 | * @param baseDir the base directory into which the ReactiveJournal will create a ReactiveJournal-dedicated data directory {@value REACTIVE_JOURNAL_DIRNAME}. 42 | * @see #getDir() 43 | * @see #clearCache() 44 | */ 45 | public ReactiveJournal(String baseDir){ 46 | this.dir = Paths.get(baseDir, REACTIVE_JOURNAL_DIRNAME).toString(); 47 | } 48 | 49 | /** 50 | * Create an {@link ReactiveJournal} that will store or read its data from the {@literal journalName} directory 51 | * inside the given {@literal baseDir}. Use {@link #getDir()} to obtain the full path of said directory, and 52 | * {@link #clearCache()} to clean that directory of ReactiveJournal-specific files. 53 | * 54 | * @param baseDir the base directory into which the ReactiveJournal will create a ReactiveJournal-dedicated data directory. 55 | * @param journalName the name to use for this ReactiveJournal's data directory, instead of the default {@value REACTIVE_JOURNAL_DIRNAME}. 56 | * @see #getDir() 57 | * @see #clearCache() 58 | */ 59 | public ReactiveJournal(String baseDir, String journalName){ 60 | this.dir = Paths.get(baseDir, journalName).toString(); 61 | } 62 | 63 | public ReactiveRecorder createReactiveRecorder(){ 64 | return new ReactiveRecorder(this); 65 | } 66 | 67 | public ReactivePlayer createReactivePlayer(){ 68 | return new ReactivePlayer(this); 69 | } 70 | 71 | public ReactiveValidator createReactiveValidator(){ 72 | return new ReactiveValidator(); 73 | } 74 | 75 | /** 76 | * Return the path to the ReactiveJournal-specific directory used to store binary representation of the journal on disk. 77 | * This directory is one level deeper than the base directory initially passed to ReactiveJournal's 78 | * {@link ReactiveJournal#ReactiveJournal(String) constructor}. 79 | * 80 | * @return the path to the ReactiveJournal specific directory. 81 | */ 82 | public String getDir() { 83 | return dir; 84 | } 85 | 86 | /** 87 | * Clear this ReactiveJournal's {@link #getDir() data directory} from ReactiveJournal-specific binary files, and remove it as 88 | * well (but only if it has become empty). If other files have been created inside the data directory, it is the 89 | * responsibility of the user to delete them AND the data directory. 90 | * 91 | * @throws IOException in case of directory traversal or file deletion problems 92 | */ 93 | public void clearCache() throws IOException { 94 | LOG.info("Deleting existing recording [{}]", dir); 95 | Path journalDir = Paths.get(dir); 96 | if(Files.exists(journalDir)) { 97 | Files.walk(journalDir) 98 | .map(Path::toFile) 99 | .filter(f -> f.isFile() && f.getName().endsWith(".cq4")) 100 | .peek(f -> LOG.info("Removing {}", f.getName())) 101 | .forEach(File::delete); 102 | 103 | try { 104 | Files.deleteIfExists(journalDir); 105 | } catch (DirectoryNotEmptyException e) { 106 | LOG.info("Directory does not only contain cq4 files, not deleted"); 107 | } 108 | } 109 | } 110 | 111 | public void writeToFile(String fileOutput){ 112 | writeToFile(fileOutput, false, null); 113 | } 114 | public void writeToFile(String fileOutput, boolean toStdout){ 115 | writeToFile(fileOutput, toStdout, null); 116 | } 117 | 118 | /** 119 | * Writes the journal in a human readable form to a file. Optionally also writes it to stdout. 120 | * @param fileOutput The name of the file 121 | * @param toStdout Whether it should be written to stdout 122 | * @param zoneId TimeZone to display to format the time. If null uses millis since 1970. 123 | */ 124 | public void writeToFile(String fileOutput, boolean toStdout, ZoneId zoneId) { 125 | LOG.info("Writing recording to dir [" + fileOutput + "]"); 126 | try (ChronicleQueue queue = createQueue()) { 127 | ExcerptTailer tailer = queue.createTailer(); 128 | try { 129 | writeQueueToFile(tailer, fileOutput, toStdout, zoneId); 130 | } catch (IOException e) { 131 | LOG.error("Error writing to file", e); 132 | } 133 | } 134 | LOG.info("Writing to dir complete"); 135 | } 136 | 137 | //todo should this cache the queue - probably 138 | ChronicleQueue createQueue(){ 139 | int blockSize = Integer.getInteger("chronicle.queueBlockSize", -1); 140 | ChronicleQueue queue = null; 141 | if(blockSize==-1) { 142 | queue = SingleChronicleQueueBuilder.binary(dir).build(); 143 | }else { 144 | queue = SingleChronicleQueueBuilder.binary(dir).blockSize(blockSize).build(); 145 | } 146 | return queue; 147 | } 148 | 149 | AtomicLong getMessageCounter() { 150 | return messageCounter; 151 | } 152 | 153 | private static void writeQueueToFile(ExcerptTailer tailer, String fileName, boolean toStdout, ZoneId zoneId) 154 | throws IOException { 155 | FileWriter fileWriter = new FileWriter(fileName); 156 | tailer.toStart(); 157 | DataItemProcessor dim = new DataItemProcessor(); 158 | while(tailer.readDocument( 159 | w -> { 160 | ValueIn in = w.getValueIn(); 161 | dim.process(in,null); 162 | String time = null; 163 | if(zoneId != null) { 164 | LocalDateTime dateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(dim.getTime()), zoneId); 165 | time = dateTime.toString(); 166 | }else{ 167 | time = String.valueOf(dim.getTime()); 168 | } 169 | try { 170 | String item = ReactiveStatus.toString(dim.getStatus()) + "\t" + dim.getMessageCount() + "\t" + time + "\t" 171 | + dim.getFilter() + "\t" + dim.getObject(); 172 | fileWriter.write(item + "\n"); 173 | if(toStdout) { 174 | LOG.info(item); 175 | } 176 | } catch (IOException e) { 177 | LOG.error("Problem writing to file[" + fileName + "]", e); 178 | } 179 | })); 180 | fileWriter.close(); 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/impl/ReactivePlayer.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import io.reactivex.internal.util.BackpressureHelper; 4 | import net.openhft.chronicle.queue.ChronicleQueue; 5 | import net.openhft.chronicle.queue.ExcerptTailer; 6 | import net.openhft.chronicle.wire.ValueIn; 7 | import org.reactivejournal.impl.PlayOptions.ReplayRate; 8 | import org.reactivejournal.util.DSUtil; 9 | import org.reactivestreams.Publisher; 10 | import org.reactivestreams.Subscriber; 11 | import org.reactivestreams.Subscription; 12 | 13 | import java.util.concurrent.ExecutorService; 14 | import java.util.concurrent.Executors; 15 | import java.util.concurrent.atomic.AtomicLong; 16 | 17 | /** 18 | * Class to playback data recorded into ReactiveJournal. 19 | */ 20 | public class ReactivePlayer { 21 | private ReactiveJournal reactiveJournal; 22 | 23 | ReactivePlayer(ReactiveJournal reactiveJournal) { 24 | this.reactiveJournal = reactiveJournal; 25 | } 26 | 27 | /** 28 | * See documentation on {@link PlayOptions} 29 | * 30 | * @param options Options controlling how play is executed. 31 | */ 32 | public Publisher play(PlayOptions options) { 33 | options.validate(); 34 | 35 | return new PlayPublisher(reactiveJournal, options); 36 | } 37 | 38 | final class PlayPublisher implements Publisher { 39 | 40 | private final ReactiveJournal reactiveJournal; 41 | private final PlayOptions options; 42 | 43 | PlayPublisher(ReactiveJournal reactiveJournal, PlayOptions options) { 44 | this.reactiveJournal = reactiveJournal; 45 | this.options = options; 46 | } 47 | 48 | @Override 49 | public void subscribe(Subscriber s) { 50 | s.onSubscribe(new PlaySubscription(s, reactiveJournal, options)); 51 | } 52 | } 53 | 54 | private final class PlaySubscription implements Subscription { 55 | private final AtomicLong requested = new AtomicLong(0); 56 | private ChronicleQueue queue; 57 | private Subscriber subscriber; 58 | private PlayOptions options; 59 | private volatile boolean cancelled = false; 60 | private final ExcerptTailer tailer; 61 | private final DataItemProcessor dim; 62 | private ExecutorService executorService; 63 | 64 | PlaySubscription(Subscriber subscriber, ReactiveJournal journal, PlayOptions options) { 65 | this.queue = journal.createQueue(); 66 | this.subscriber = subscriber; 67 | this.options = options; 68 | this.tailer = queue.createTailer(); 69 | this.dim = new DataItemProcessor(); 70 | } 71 | 72 | @Override 73 | public void request(long n) { 74 | if (!options.sameThread()) { 75 | if(executorService == null) { 76 | executorService = Executors.newSingleThreadExecutor(runnable -> { 77 | Thread thread = new Thread(runnable); 78 | thread.setDaemon(true); 79 | thread.setName("Subscription Runner [" + System.currentTimeMillis() + "]"); 80 | return thread; 81 | }); 82 | executorService.submit(() -> { 83 | while (true) { 84 | if (drain()) { 85 | return; 86 | } 87 | Thread.yield(); 88 | } 89 | }); 90 | } 91 | BackpressureHelper.add(requested, n); 92 | } else { 93 | if (n > 0) { 94 | if (BackpressureHelper.add(requested, n) == 0) { 95 | drain(); 96 | } 97 | } 98 | } 99 | } 100 | 101 | 102 | @Override 103 | public void cancel() { 104 | cancelled = true; 105 | } 106 | 107 | /* 108 | * Returns whether the subscription has terminated or not 109 | */ 110 | boolean drain() { 111 | //System.out.println("1 enter drain"); 112 | long[] lastTime = new long[]{Long.MIN_VALUE}; 113 | 114 | 115 | while (requested.get() != 0) { // don't emit more than requested 116 | if (cancelled) { 117 | return true; 118 | } 119 | 120 | boolean empty = nextItemFromQueue(tailer, options, dim); 121 | //System.out.println("2. Next item from was empty? " + empty); 122 | if (empty) { 123 | if (options.completeAtEndOfFile()) { 124 | subscriber.onComplete(); 125 | //System.out.println("Complete at end of file"); 126 | return true; 127 | } 128 | //wait for next event to appear in the journal 129 | pauseStrategy(options); 130 | continue; 131 | } 132 | 133 | if (!itemMatchingFilter(options, dim)) { 134 | // System.out.println("3. Item did not match filter"); 135 | //wait for next matching event to appear in the journal 136 | pauseStrategy(options); 137 | continue; 138 | } 139 | 140 | //System.out.println("4. About to pause for actual time"); 141 | if (options.replayRate() == ReplayRate.ACTUAL_TIME) { 142 | pauseForActualTime(lastTime, dim); 143 | } 144 | //System.out.println("5. After to pause for actual time"); 145 | 146 | //is there an error 147 | if (dim.getStatus() == ReactiveStatus.ERROR) { 148 | //System.out.println("5. Error"); 149 | subscriber.onError((Throwable) dim.getObject()); 150 | return true; 151 | } 152 | 153 | //is the request complete 154 | if (isComplete(options, dim)) { 155 | // System.out.println("5. Complete"); 156 | subscriber.onComplete(); 157 | return true; 158 | } 159 | 160 | //System.out.println("6. returning object " + dim.getObject() + " requested " + requested); 161 | subscriber.onNext((T) dim.getObject()); 162 | requested.decrementAndGet(); 163 | } 164 | return false; 165 | // if a concurrent getAndIncrement() happened, we loop back and continue 166 | } 167 | 168 | private void pauseStrategy(PlayOptions options) { 169 | if(options.pauseStrategy()== PlayOptions.PauseStrategy.YIELD){ 170 | Thread.yield(); 171 | } 172 | } 173 | 174 | } 175 | 176 | private boolean isComplete(PlayOptions options, DataItemProcessor dim) { 177 | return dim.getStatus() == ReactiveStatus.COMPLETE 178 | || dim.getTime() > options.playUntilTime() 179 | || dim.getMessageCount() >= options.playUntilSeqNo(); 180 | } 181 | 182 | private boolean itemMatchingFilter(PlayOptions options, DataItemProcessor dim) { 183 | return options.filter().equals(dim.getFilter()) 184 | && !(dim.getTime() < options.playFromTime() 185 | || dim.getMessageCount() < options.playFromSeqNo()); 186 | 187 | } 188 | 189 | private void pauseForActualTime(long[] lastTime, DataItemProcessor dim) { 190 | if (lastTime[0] != Long.MIN_VALUE) { 191 | DSUtil.sleep((int) (dim.getTime() - lastTime[0])); 192 | } 193 | lastTime[0] = dim.getTime(); 194 | } 195 | 196 | /* 197 | * @return Whether the queue is empty or not 198 | */ 199 | private boolean nextItemFromQueue(ExcerptTailer tailer, 200 | PlayOptions options, DataItemProcessor dim) { 201 | return !tailer.readDocument(w -> { 202 | ValueIn in = w.getValueIn(); 203 | dim.process(in, options.using()); 204 | }); 205 | } 206 | } -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/impl/ReactiveRecorder.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import net.openhft.chronicle.queue.ChronicleQueue; 4 | import net.openhft.chronicle.queue.ExcerptAppender; 5 | import net.openhft.chronicle.wire.WireOut; 6 | import org.reactivestreams.Publisher; 7 | import org.reactivestreams.Subscriber; 8 | import org.reactivestreams.Subscription; 9 | import org.reactivejournal.util.TriConsumer; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.util.function.BiConsumer; 14 | 15 | /** 16 | * Class to record input into ReactiveJournal. 17 | */ 18 | public class ReactiveRecorder { 19 | private static final Logger LOG = LoggerFactory.getLogger(ReactiveRecorder.class.getName()); 20 | 21 | private ReactiveJournal reactiveJournal; 22 | 23 | public ReactiveRecorder(ReactiveJournal reactiveJournal) { 24 | this.reactiveJournal = reactiveJournal; 25 | } 26 | 27 | public void recordAsync(Publisher publisher, String filter){ 28 | new Thread(()->record(publisher,filter)).start(); 29 | } 30 | 31 | 32 | private TriConsumer getOnNextConsumerRecorder(){ 33 | return (a, f, v) -> a.writeDocument(w -> { 34 | writeObject(w, f, v, ReactiveStatus.VALID); 35 | }); 36 | } 37 | 38 | private BiConsumer getOnCompleteRecorder(){ 39 | return (a,f) -> a.writeDocument(w -> { 40 | writeObject(w, f, new EndOfStream(), ReactiveStatus.COMPLETE); 41 | LOG.debug("Adding end of stream token"); 42 | }); 43 | } 44 | 45 | private TriConsumer getOnErrorRecorder(){ 46 | return (a, f, t) -> a.writeDocument(w -> { 47 | writeObject(w, f, t, ReactiveStatus.ERROR); 48 | }); 49 | } 50 | 51 | private void writeObject(WireOut wireOut, String filter, Object obj, byte status){ 52 | wireOut.getValueOut().int8(status); 53 | wireOut.getValueOut().int64(reactiveJournal.getMessageCounter().incrementAndGet()); 54 | wireOut.getValueOut().int64(System.currentTimeMillis()); 55 | wireOut.getValueOut().text(filter); 56 | if(status== ReactiveStatus.ERROR){ 57 | wireOut.getValueOut().throwable((Throwable)obj); 58 | }else { 59 | wireOut.getValueOut().object(obj); 60 | } 61 | } 62 | 63 | public void record(Publisher publisher, String filter) { 64 | ChronicleQueue queue = reactiveJournal.createQueue(); 65 | ExcerptAppender appender = queue.acquireAppender(); 66 | 67 | TriConsumer onNextConsumer = getOnNextConsumerRecorder(); 68 | BiConsumer onCompleteConsumer = getOnCompleteRecorder(); 69 | TriConsumer onErrorConsumer = getOnErrorRecorder(); 70 | 71 | publisher.subscribe(new Subscriber() { 72 | @Override 73 | public void onSubscribe(Subscription subscription) { 74 | subscription.request(Long.MAX_VALUE); 75 | } 76 | 77 | @Override 78 | public void onNext(Object o) { 79 | onNextConsumer.accept(appender, filter, o); 80 | } 81 | 82 | @Override 83 | public void onError(Throwable throwable) { 84 | onErrorConsumer.accept(appender, filter, throwable); 85 | } 86 | 87 | @Override 88 | public void onComplete() { 89 | onCompleteConsumer.accept(appender, filter); 90 | } 91 | }); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/impl/ReactiveStatus.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | /** 4 | * Created by daniel on 18/05/17. 5 | */ 6 | public interface ReactiveStatus { 7 | byte VALID = 0; 8 | byte ERROR = 1; 9 | byte COMPLETE = 2; 10 | 11 | static String toString(byte status){ 12 | switch (status){ 13 | case VALID: 14 | return "VALID"; 15 | case ERROR: 16 | return "ERROR"; 17 | case COMPLETE: 18 | return "COMPLETE"; 19 | default: 20 | return "UNKNOWN"; 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/impl/ReactiveValidator.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import net.openhft.chronicle.queue.ChronicleQueue; 4 | import net.openhft.chronicle.queue.ExcerptTailer; 5 | import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder; 6 | import net.openhft.chronicle.wire.ValueIn; 7 | import org.reactivestreams.Publisher; 8 | import org.reactivestreams.Subscriber; 9 | import org.reactivestreams.Subscription; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | /** 14 | * Class to help in validating data in rxJournal during testing. 15 | */ 16 | public class ReactiveValidator { 17 | private static final Logger LOG = LoggerFactory.getLogger(ReactiveValidator.class.getName()); 18 | private ValidationResult validationResult; 19 | private DataItemProcessor dataItemProcessor = new DataItemProcessor(); 20 | 21 | public void validate(String fileName, Publisher flowable, String filter, Subscriber subscriber) { 22 | ChronicleQueue queue = SingleChronicleQueueBuilder.binary(fileName).build(); 23 | ExcerptTailer tailer = queue.createTailer(); 24 | validationResult = new ValidationResult(); 25 | 26 | flowable.subscribe(new Subscriber() { 27 | @Override 28 | public void onSubscribe(Subscription subscription) { 29 | subscription.request(Long.MAX_VALUE); 30 | } 31 | 32 | @Override 33 | public void onNext(Object o) { 34 | Object onQueue = null; 35 | try { 36 | onQueue = getNextMatchingFilter(tailer, filter); 37 | }catch(IllegalStateException e){ 38 | subscriber.onComplete(); 39 | }catch(Exception e){ 40 | subscriber.onError(e); 41 | } 42 | 43 | if (onQueue.equals(o)) { 44 | validationResult.setResult(ValidationResult.Result.OK); 45 | } else { 46 | validationResult.setResult(ValidationResult.Result.BAD); 47 | } 48 | validationResult.setFromQueue(onQueue); 49 | validationResult.setGenerated(o); 50 | subscriber.onNext(validationResult); 51 | } 52 | 53 | @Override 54 | public void onError(Throwable throwable) { 55 | LOG.error("error in validate [{}]", throwable); 56 | } 57 | 58 | @Override 59 | public void onComplete() { 60 | subscriber.onComplete(); 61 | queue.close(); 62 | } 63 | }); 64 | } 65 | 66 | 67 | private Object getNextMatchingFilter(ExcerptTailer tailer, String filter) { 68 | nextItemFromQueue(tailer, dataItemProcessor); 69 | if (dataItemProcessor.getFilter().equals(filter)) { 70 | return dataItemProcessor.getObject(); 71 | } else { 72 | return getNextMatchingFilter(tailer, filter); 73 | } 74 | 75 | } 76 | 77 | private boolean nextItemFromQueue(ExcerptTailer tailer,DataItemProcessor dim) { 78 | return !tailer.readDocument(w -> { 79 | ValueIn in = w.getValueIn(); 80 | dim.process(in, null); 81 | }); 82 | } 83 | 84 | public ValidationResult getValidationResult() { 85 | return validationResult; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/impl/ValidationResult.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | /** 4 | * ValidationResult holds the results of a validation carried out by ReactiveValidator 5 | */ 6 | public class ValidationResult { 7 | public enum Result {OK, BAD} 8 | 9 | private Result result; 10 | private T fromQueue, generated; 11 | private Result summaryResult = null; 12 | private int totalCount, totalFailures, totalCorrect; 13 | 14 | public Result getResult() { 15 | return result; 16 | } 17 | 18 | void setResult(Result result) { 19 | totalCount++; 20 | this.result = result; 21 | if(result == Result.BAD){ 22 | summaryResult = Result.BAD; 23 | totalFailures++; 24 | }else{ 25 | if(summaryResult==null){ 26 | summaryResult = Result.OK; 27 | } 28 | totalCorrect++; 29 | } 30 | } 31 | 32 | public T getItemFromQueue() { 33 | return fromQueue; 34 | } 35 | 36 | public void setFromQueue(T fromQueue) { 37 | this.fromQueue = fromQueue; 38 | } 39 | 40 | public T getItemObservable() { 41 | return generated; 42 | } 43 | 44 | public void setGenerated(T generated) { 45 | this.generated = generated; 46 | } 47 | 48 | public Result summaryResult(){ 49 | return summaryResult; 50 | } 51 | 52 | public int summaryItemsCompared(){ 53 | return totalCount; 54 | } 55 | 56 | public int summaryItemsValid(){ 57 | return totalCorrect; 58 | } 59 | 60 | public int summmaryItemsFailed(){ 61 | return totalFailures; 62 | } 63 | 64 | @Override 65 | public String toString() { 66 | return "ValidationResult{" + 67 | "result=" + result + 68 | ", fromQueue=" + fromQueue + 69 | ", generated=" + generated + 70 | ", summaryResult=" + summaryResult + 71 | ", totalCount=" + totalCount + 72 | ", totalFailures=" + totalFailures + 73 | ", totalCorrect=" + totalCorrect + 74 | '}'; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/impl/rxjava/RxJavaPlayer.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl.rxjava; 2 | 3 | import io.reactivex.Flowable; 4 | import org.reactivejournal.impl.PlayOptions; 5 | import org.reactivejournal.impl.ReactiveJournal; 6 | import org.reactivejournal.impl.ReactivePlayer; 7 | 8 | /** 9 | * A specific RxJava implementation of ReactivePlayer. 10 | */ 11 | public class RxJavaPlayer { 12 | private ReactivePlayer reactivePlayer; 13 | 14 | public RxJavaPlayer(ReactiveJournal reactiveJournal){ 15 | reactivePlayer = reactiveJournal.createReactivePlayer(); 16 | } 17 | 18 | public Flowable play(PlayOptions options){ 19 | return Flowable.fromPublisher(reactivePlayer.play(options)); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/util/DSUtil.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.util; 2 | 3 | /** 4 | * Created by daniel on 27/04/17. 5 | */ 6 | public class DSUtil { 7 | public static void sleep(int ms) { 8 | try { 9 | Thread.sleep(ms); 10 | } catch (InterruptedException e) { 11 | e.printStackTrace(); 12 | } 13 | } 14 | 15 | public static void exitAfter(int timeMs) { 16 | new Thread(()->{ 17 | sleep(timeMs); 18 | System.out.println("Exiting after waiting " + timeMs + "ms"); 19 | System.exit(0); 20 | }).start(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/reactivejournal/util/TriConsumer.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.util; 2 | 3 | /** 4 | * Created by daniel on 10/05/17. 5 | */ 6 | @FunctionalInterface 7 | public interface TriConsumer { 8 | /** 9 | * Performs this operation on the given arguments. 10 | * 11 | * @param a the first input argument 12 | * @param b the second input argument 13 | * @param c the third input argument 14 | */ 15 | void accept(A a, B b, C c); 16 | } 17 | -------------------------------------------------------------------------------- /src/playground/java/queue/MarketData.java: -------------------------------------------------------------------------------- 1 | package queue; 2 | 3 | import net.openhft.chronicle.wire.Marshallable; 4 | 5 | /** 6 | * Created by daniel on 06/12/16. 7 | */ 8 | public class MarketData implements Marshallable { 9 | private double bidPrice; 10 | private double askPrice; 11 | private int volume; 12 | private String id; 13 | 14 | public MarketData(){ 15 | 16 | } 17 | 18 | public MarketData(String id, int volume, double bidPrice, double askPrice) { 19 | this.bidPrice = bidPrice; 20 | this.askPrice = askPrice; 21 | this.volume = volume; 22 | this.id = id; 23 | } 24 | 25 | public double getAskPrice() { 26 | return askPrice; 27 | } 28 | 29 | public void setAskPrice(double askPrice) { 30 | this.askPrice = askPrice; 31 | } 32 | 33 | public double getBidPrice() { 34 | return bidPrice; 35 | } 36 | 37 | public void setBidPrice(double bidPrice) { 38 | this.bidPrice = bidPrice; 39 | } 40 | 41 | public int getVolume() { 42 | return volume; 43 | } 44 | 45 | public void setVolume(int volume) { 46 | this.volume = volume; 47 | } 48 | 49 | public String getId() { 50 | return id; 51 | } 52 | 53 | public void setId(String id) { 54 | this.id = id; 55 | } 56 | 57 | @Override 58 | public String toString() { 59 | return "MarketData{" + 60 | "bidPrice=" + bidPrice + 61 | ", askPrice=" + askPrice + 62 | ", volume=" + volume + 63 | ", id='" + id + '\'' + 64 | '}'; 65 | } 66 | 67 | @Override 68 | public boolean equals(Object o) { 69 | if (this == o) return true; 70 | if (o == null || getClass() != o.getClass()) return false; 71 | 72 | MarketData that = (MarketData) o; 73 | 74 | if (Double.compare(that.bidPrice, bidPrice) != 0) return false; 75 | if (Double.compare(that.askPrice, askPrice) != 0) return false; 76 | if (volume != that.volume) return false; 77 | return id != null ? id.equals(that.id) : that.id == null; 78 | } 79 | 80 | @Override 81 | public int hashCode() { 82 | int result; 83 | long temp; 84 | temp = Double.doubleToLongBits(bidPrice); 85 | result = (int) (temp ^ (temp >>> 32)); 86 | temp = Double.doubleToLongBits(askPrice); 87 | result = 31 * result + (int) (temp ^ (temp >>> 32)); 88 | result = 31 * result + volume; 89 | result = 31 * result + (id != null ? id.hashCode() : 0); 90 | return result; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/playground/java/queue/MarshallableHolder.java: -------------------------------------------------------------------------------- 1 | package queue; 2 | 3 | import net.openhft.chronicle.wire.Marshallable; 4 | 5 | /** 6 | * Created by daniel on 27/04/17. 7 | */ 8 | public class MarshallableHolder implements Marshallable { 9 | Object held; 10 | 11 | public MarshallableHolder(Object held) { 12 | this.held = held; 13 | } 14 | 15 | @Override 16 | public String toString() { 17 | return "MarshallableHolder{" + 18 | "held=" + held + 19 | '}'; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/playground/java/queue/PauseTest.java: -------------------------------------------------------------------------------- 1 | package queue; 2 | 3 | import java.util.concurrent.locks.LockSupport; 4 | 5 | /** 6 | * Created by daniel on 25/05/17. 7 | */ 8 | public class PauseTest { 9 | public static void main(String[] args) throws InterruptedException { 10 | long startTime = System.nanoTime(); 11 | int spins = 10_000; 12 | for (int i = 0; i < spins; i++) { 13 | Thread.yield(); 14 | //LockSupport.parkNanos(1); 15 | //Thread.sleep(1); 16 | } 17 | System.out.println((System.nanoTime()-startTime)/spins); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/playground/java/queue/QueuePlayGround.java: -------------------------------------------------------------------------------- 1 | package queue; 2 | 3 | import net.openhft.chronicle.core.UnsafeMemory; 4 | import net.openhft.chronicle.queue.ChronicleQueue; 5 | import net.openhft.chronicle.queue.ExcerptAppender; 6 | import net.openhft.chronicle.queue.ExcerptTailer; 7 | import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder; 8 | import net.openhft.chronicle.wire.ValueIn; 9 | import org.reactivejournal.util.DSUtil; 10 | import sun.misc.Unsafe; 11 | 12 | import java.nio.charset.StandardCharsets; 13 | import java.util.concurrent.Executors; 14 | 15 | /** 16 | * Created by daniel on 19/10/16. 17 | */ 18 | public class QueuePlayGround { 19 | private static String file = "/tmp/playground"; 20 | 21 | public static void main(String[] args) { 22 | writeThrowableToQueue(); 23 | } 24 | 25 | public static void writeThrowableToQueue(){ 26 | final Throwable throwable; 27 | try { 28 | throw new RuntimeException("serialise"); 29 | }catch(Throwable e){ 30 | throwable = e; 31 | } 32 | 33 | try (ChronicleQueue queue = SingleChronicleQueueBuilder.binary(file).build()) { 34 | final ExcerptAppender appender = queue.acquireAppender(); 35 | 36 | appender.writeBytes(b -> { 37 | long address = b.address(b.writePosition()); 38 | Unsafe unsafe = UnsafeMemory.UNSAFE; 39 | unsafe.putByte(address, (byte) 0x12); 40 | address += 1; 41 | unsafe.putInt(address, 0x345678); 42 | address += 4; 43 | unsafe.putLong(address, 0x999000999000L); 44 | address += 8; 45 | byte[] bytes = "Hello World".getBytes(StandardCharsets.ISO_8859_1); 46 | unsafe.putByte(address, (byte)bytes.length); 47 | address += 1; 48 | unsafe.copyMemory(bytes, Unsafe.ARRAY_BYTE_BASE_OFFSET, null, address, bytes.length); 49 | b.writeSkip(1 + 4 + 8 + bytes.length); 50 | }); 51 | 52 | 53 | final ExcerptTailer tailer = queue.createTailer(); 54 | tailer.readBytes(b -> { 55 | long address = b.address(b.readPosition()); 56 | Unsafe unsafe = UnsafeMemory.UNSAFE; 57 | int code = unsafe.getByte(address); 58 | address++; 59 | int num = unsafe.getInt(address); 60 | address += 4; 61 | long num2 = unsafe.getLong(address); 62 | address += 8; 63 | int length = unsafe.getByte(address); 64 | address++; 65 | byte[] bytes = new byte[length]; 66 | unsafe.copyMemory(null, address, bytes, Unsafe.ARRAY_BYTE_BASE_OFFSET, bytes.length); 67 | String text = new String(bytes, StandardCharsets.ISO_8859_1); 68 | System.out.println(text); 69 | // do something with values 70 | }); 71 | } 72 | 73 | } 74 | 75 | public static void writeNonSerial() { 76 | //for(int t=0; t<5; t++) { 77 | // new Thread(() -> { 78 | try (ChronicleQueue queue = SingleChronicleQueueBuilder.binary(file).bufferCapacity(1000).build()) { 79 | final ExcerptAppender appender = queue.acquireAppender(); 80 | for (int i = 0; i < 10; i++) { 81 | final int f = i; 82 | appender.writeDocument(w -> { 83 | w.getValueOut().int64(System.currentTimeMillis()); 84 | w.getValueOut().object(new MarketData()); 85 | }); 86 | DSUtil.sleep(100); 87 | } 88 | 89 | } 90 | // }).start(); 91 | // } 92 | } 93 | 94 | public static void writeToQueue() { 95 | //for(int t=0; t<5; t++) { 96 | // new Thread(() -> { 97 | try (ChronicleQueue queue = SingleChronicleQueueBuilder.binary(file).build()) { 98 | final ExcerptAppender appender = queue.acquireAppender(); 99 | for (int i = 0; i < 10; i++) { 100 | final int f = i; 101 | appender.writeDocument(w -> { 102 | w.getValueOut().int64(System.currentTimeMillis()); 103 | w.getValueOut().object(new MarketData()); 104 | }); 105 | DSUtil.sleep(100); 106 | } 107 | 108 | } 109 | // }).start(); 110 | // } 111 | } 112 | 113 | public static void readFromQueue(){ 114 | 115 | ChronicleQueue queue = SingleChronicleQueueBuilder.binary(file).build(); 116 | 117 | final ExcerptTailer tailer = queue.createTailer(); 118 | //System.out.println(queue.dump()); 119 | Executors.newSingleThreadExecutor().submit(()-> { 120 | while (true) { 121 | tailer.readDocument(w -> { 122 | ValueIn in = w.getValueIn(); 123 | long time = in.int64(); 124 | String filter = in.text(); 125 | MarketData trade = in.object(new MarketData(), MarketData.class); 126 | System.out.println(time + "->" + trade); 127 | }); 128 | } 129 | }); 130 | 131 | } 132 | 133 | public static void dumpQueue(){ 134 | 135 | try(ChronicleQueue queue = SingleChronicleQueueBuilder.binary(file).build()){ 136 | System.out.println(queue.dump()); 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /src/playground/java/queue/TestStore.java: -------------------------------------------------------------------------------- 1 | package queue; 2 | 3 | /** 4 | * Created by daniel on 27/04/17. 5 | */ 6 | public class TestStore { 7 | String s; 8 | int i; 9 | 10 | public TestStore(String s, int i) { 11 | this.s = s; 12 | this.i = i; 13 | } 14 | 15 | public String getS() { 16 | return s; 17 | } 18 | 19 | public void setS(String s) { 20 | this.s = s; 21 | } 22 | 23 | public int getI() { 24 | return i; 25 | } 26 | 27 | public void setI(int i) { 28 | this.i = i; 29 | } 30 | 31 | @Override 32 | public String toString() { 33 | return "TestStore{" + 34 | "s='" + s + '\'' + 35 | ", i=" + i + 36 | '}'; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/org/reactivejournal/impl/ReactiveErrorTest.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import io.reactivex.BackpressureStrategy; 4 | import io.reactivex.Flowable; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 8 | 9 | import java.io.IOException; 10 | import java.time.ZoneId; 11 | import java.util.concurrent.atomic.AtomicInteger; 12 | 13 | 14 | /** 15 | * Created by daniel on 17/05/17. 16 | */ 17 | public class ReactiveErrorTest { 18 | @Test 19 | public void errorTest() throws IOException{ 20 | //publish a couple of items then an eeror 21 | //make sure the error is received onError() 22 | 23 | //try a couple of filters one with an error and one without 24 | //one should end with onError and one with onComplete() 25 | //Create the reactiveRecorder and delete any previous content by clearing the cache 26 | Throwable rte = new RuntimeException("Test Error"); 27 | Flowable errorFlowable = Flowable.create( 28 | e -> { 29 | e.onNext("one"); 30 | e.onNext("two"); 31 | e.onError(rte); 32 | }, 33 | BackpressureStrategy.BUFFER 34 | ); 35 | 36 | Flowable validFlowable = Flowable.create( 37 | e -> { 38 | e.onNext(100); 39 | e.onNext(200); 40 | e.onComplete(); 41 | }, 42 | BackpressureStrategy.BUFFER 43 | ); 44 | 45 | ReactiveJournal reactiveJournal = new ReactiveJournal("/tmp/testError"); 46 | reactiveJournal.writeToFile("/tmp/testError/error.txt",true, ZoneId.systemDefault()); 47 | reactiveJournal.clearCache(); 48 | 49 | //Pass the input stream into the reactiveRecorder which will subscribe to it and record all events. 50 | //The subscription will not be activated on a new thread which will allow this program to continue. 51 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 52 | reactiveRecorder.recordAsync(errorFlowable, "errorinput"); 53 | reactiveRecorder.recordAsync(validFlowable, "validinput"); 54 | 55 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 56 | PlayOptions options = new PlayOptions().filter("errorinput").sameThread(true); 57 | Flowable recordedObservable = rxPlayer.play(options); 58 | 59 | AtomicInteger onNext = new AtomicInteger(0); 60 | AtomicInteger onComplete = new AtomicInteger(0); 61 | AtomicInteger onError = new AtomicInteger(0); 62 | Throwable[] tArray = new Throwable[1]; 63 | 64 | //Pass the output stream (of words) into the reactiveRecorder which will subscribe to it and record all events. 65 | recordedObservable.subscribe(i->onNext.incrementAndGet(), 66 | e->{ 67 | onError.incrementAndGet(); 68 | tArray[0] = (Throwable) e; 69 | }, 70 | ()->onComplete.incrementAndGet()); 71 | 72 | Assert.assertEquals(2, onNext.get()); 73 | Assert.assertEquals(1, onError.get()); 74 | Assert.assertEquals(0, onComplete.get()); 75 | Assert.assertEquals(rte.getMessage(), tArray[0].getMessage()); 76 | Assert.assertEquals(rte.getClass(), tArray[0].getClass()); 77 | Assert.assertEquals(rte.getStackTrace()[0], tArray[0].getStackTrace()[0]); 78 | 79 | rxPlayer = new RxJavaPlayer(reactiveJournal); 80 | options = new PlayOptions().filter("validinput").sameThread(true); 81 | recordedObservable = rxPlayer.play(options); 82 | 83 | AtomicInteger onNextValid = new AtomicInteger(0); 84 | AtomicInteger onCompleteValid = new AtomicInteger(0); 85 | AtomicInteger onErrorValid = new AtomicInteger(0); 86 | 87 | //Pass the output stream (of words) into the reactiveRecorder which will subscribe to it and record all events. 88 | recordedObservable.subscribe(i->onNextValid.incrementAndGet(), 89 | e->{ 90 | onErrorValid.incrementAndGet(); 91 | }, 92 | ()->onCompleteValid.incrementAndGet()); 93 | 94 | Assert.assertEquals(2, onNextValid.get()); 95 | Assert.assertEquals(0, onErrorValid.get()); 96 | Assert.assertEquals(1, onCompleteValid.get()); 97 | 98 | reactiveJournal.writeToFile("/tmp/testError/error.txt",true); 99 | 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/test/java/org/reactivejournal/impl/ReactiveFromUntilSeqNoTest.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import io.reactivex.BackpressureStrategy; 4 | import io.reactivex.Flowable; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 8 | 9 | import java.io.IOException; 10 | import java.util.concurrent.atomic.AtomicInteger; 11 | 12 | 13 | /** 14 | * Created by daniel on 17/05/17. 15 | */ 16 | public class ReactiveFromUntilSeqNoTest { 17 | @Test 18 | public void fromUntilTest() throws IOException { 19 | Flowable errorFlowable = Flowable.create( 20 | e -> { 21 | e.onNext("one"); 22 | e.onNext("two"); 23 | e.onNext("three"); 24 | e.onComplete(); 25 | }, 26 | BackpressureStrategy.BUFFER 27 | ); 28 | 29 | 30 | ReactiveJournal reactiveJournal = new ReactiveJournal("/tmp/testFromUntil"); 31 | reactiveJournal.clearCache(); 32 | 33 | //Pass the input stream into the reactiveRecorder which will subscribe to it and record all events. 34 | //The subscription will not be activated on a new thread which will allow this program to continue. 35 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 36 | reactiveRecorder.recordAsync(errorFlowable, "fromuntil"); 37 | 38 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 39 | PlayOptions options = new PlayOptions().filter("fromuntil").sameThread(true); 40 | Flowable recordedObservable = rxPlayer.play(options); 41 | 42 | AtomicInteger onNext = new AtomicInteger(0); 43 | AtomicInteger onComplete = new AtomicInteger(0); 44 | AtomicInteger onError = new AtomicInteger(0); 45 | 46 | //Pass the output stream (of words) into the reactiveRecorder which will subscribe to it and record all events. 47 | recordedObservable.subscribe(i -> onNext.incrementAndGet(), 48 | e -> { 49 | onError.incrementAndGet(); 50 | }, 51 | () -> onComplete.incrementAndGet()); 52 | 53 | Assert.assertEquals(3, onNext.get()); 54 | Assert.assertEquals(0, onError.get()); 55 | Assert.assertEquals(1, onComplete.get()); 56 | 57 | 58 | rxPlayer = new RxJavaPlayer(reactiveJournal); 59 | options = new PlayOptions().filter("fromuntil") 60 | .replayRate(PlayOptions.ReplayRate.FAST) 61 | .playFromSeqNo(2) 62 | .playUntilSeqNo(3) 63 | .sameThread(true); 64 | recordedObservable = rxPlayer.play(options); 65 | 66 | AtomicInteger onNextFromUntil = new AtomicInteger(0); 67 | AtomicInteger onCompleteFromUntil = new AtomicInteger(0); 68 | AtomicInteger onErrorFromUntil = new AtomicInteger(0); 69 | 70 | String[] result = new String[1]; 71 | recordedObservable.subscribe(i -> { 72 | onNextFromUntil.incrementAndGet(); 73 | result[0] = (String) i; 74 | }, 75 | e -> { 76 | onErrorFromUntil.incrementAndGet(); 77 | }, 78 | () -> onCompleteFromUntil.incrementAndGet()); 79 | 80 | reactiveJournal.writeToFile("/tmp/testError/fromUntil.txt", true); 81 | 82 | Assert.assertEquals(1, onNextFromUntil.get()); 83 | Assert.assertEquals(0, onErrorFromUntil.get()); 84 | Assert.assertEquals(1, onCompleteFromUntil.get()); 85 | Assert.assertEquals("two", result[0]); 86 | 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/test/java/org/reactivejournal/impl/ReactiveFromUntilTimeTest.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import io.reactivex.BackpressureStrategy; 4 | import io.reactivex.Flowable; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 8 | import org.reactivejournal.util.DSUtil; 9 | 10 | import java.io.IOException; 11 | import java.util.concurrent.atomic.AtomicInteger; 12 | 13 | 14 | /** 15 | * Created by daniel on 17/05/17. 16 | */ 17 | public class ReactiveFromUntilTimeTest { 18 | @Test 19 | public void fromUntilTest() throws IOException { 20 | long[] time = new long[1]; 21 | Flowable errorFlowable = Flowable.create( 22 | e -> { 23 | time[0] = System.currentTimeMillis(); 24 | e.onNext("one"); 25 | DSUtil.sleep(1000); 26 | e.onNext("two"); 27 | DSUtil.sleep(200); 28 | e.onNext("three"); 29 | e.onComplete(); 30 | }, 31 | BackpressureStrategy.BUFFER 32 | ); 33 | 34 | 35 | ReactiveJournal reactiveJournal = new ReactiveJournal("/tmp/testFromUntil"); 36 | reactiveJournal.writeToFile("/tmp/testError/error.txt", true); 37 | reactiveJournal.clearCache(); 38 | 39 | //Pass the input stream into the reactiveRecorder which will subscribe to it and record all events. 40 | //The subscription will not be activated on a new thread which will allow this program to continue. 41 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 42 | reactiveRecorder.recordAsync(errorFlowable, "fromuntil"); 43 | 44 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 45 | PlayOptions options = new PlayOptions().filter("fromuntil").sameThread(true); 46 | Flowable recordedObservable = rxPlayer.play(options); 47 | 48 | AtomicInteger onNext = new AtomicInteger(0); 49 | AtomicInteger onComplete = new AtomicInteger(0); 50 | AtomicInteger onError = new AtomicInteger(0); 51 | 52 | //Pass the output stream (of words) into the reactiveRecorder which will subscribe to it and record all events. 53 | recordedObservable.subscribe(i -> onNext.incrementAndGet(), 54 | e -> { 55 | onError.incrementAndGet(); 56 | }, 57 | () -> onComplete.incrementAndGet()); 58 | Assert.assertEquals(3, onNext.get()); 59 | Assert.assertEquals(0, onError.get()); 60 | Assert.assertEquals(1, onComplete.get()); 61 | 62 | 63 | rxPlayer = new RxJavaPlayer(reactiveJournal); 64 | options = new PlayOptions().filter("fromuntil") 65 | .replayRate(PlayOptions.ReplayRate.FAST) 66 | .playFromTime(time[0] + 200) 67 | .playUntilTime(time[0] + 1100) 68 | .sameThread(true); 69 | recordedObservable = rxPlayer.play(options); 70 | 71 | AtomicInteger onNextFromUntil = new AtomicInteger(0); 72 | AtomicInteger onCompleteFromUntil = new AtomicInteger(0); 73 | AtomicInteger onErrorFromUntil = new AtomicInteger(0); 74 | 75 | String[] result = new String[1]; 76 | recordedObservable.subscribe(i -> { 77 | onNextFromUntil.incrementAndGet(); 78 | result[0] = (String) i; 79 | }, 80 | e -> { 81 | onErrorFromUntil.incrementAndGet(); 82 | }, 83 | () -> onCompleteFromUntil.incrementAndGet()); 84 | 85 | Assert.assertEquals(1, onNextFromUntil.get()); 86 | Assert.assertEquals(0, onErrorFromUntil.get()); 87 | Assert.assertEquals(1, onCompleteFromUntil.get()); 88 | Assert.assertEquals("two", result[0]); 89 | 90 | 91 | reactiveJournal.writeToFile("/tmp/testError/error.txt", true); 92 | 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/test/java/org/reactivejournal/impl/ReactiveJournalDirectoryTest.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import io.reactivex.Flowable; 4 | import org.junit.After; 5 | import org.junit.Assert; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | 9 | import java.io.File; 10 | import java.io.IOException; 11 | import java.nio.file.Files; 12 | import java.nio.file.Path; 13 | 14 | public class ReactiveJournalDirectoryTest { 15 | 16 | Path base; 17 | Path expectedDir; 18 | 19 | @Before 20 | public void createDir() throws IOException { 21 | base = Files.createTempDirectory("ReactiveJournal"); 22 | expectedDir = base.resolve(ReactiveJournal.REACTIVE_JOURNAL_DIRNAME); 23 | } 24 | 25 | @After 26 | public void deleteDir() throws IOException { 27 | if (Files.exists(expectedDir)) { 28 | Files.walk(expectedDir) 29 | .map(Path::toFile) 30 | .filter(f -> f.isFile() && ( 31 | f.getName().endsWith(".txt") || f.getName().endsWith(".cq4"))) 32 | .peek(f -> System.out.println("will delete " + f)) 33 | .forEach(File::delete); 34 | Files.deleteIfExists(expectedDir); 35 | } 36 | Files.deleteIfExists(base); 37 | } 38 | 39 | @Test 40 | public void rxJournalUsesDefaultSubDir() throws IOException { 41 | ReactiveJournal journal = new ReactiveJournal(base.toString()); 42 | 43 | Assert.assertEquals(expectedDir.toString(), journal.getDir()); 44 | } 45 | 46 | @Test 47 | public void rxJournalUsesSpecificSubDirWhenNamed() throws IOException { 48 | ReactiveJournal journal = new ReactiveJournal(base.toString(), "foo"); 49 | 50 | Assert.assertNotEquals(expectedDir.toString(), journal.getDir()); 51 | Assert.assertEquals(expectedDir.toString().replace(ReactiveJournal.REACTIVE_JOURNAL_DIRNAME, "foo"), journal.getDir()); 52 | } 53 | 54 | @Test 55 | public void rxJournalDoesntCreateDir() { 56 | ReactiveJournal journal = new ReactiveJournal(base.toString()); 57 | 58 | Assert.assertFalse(Files.exists(expectedDir)); 59 | } 60 | 61 | @Test 62 | public void rxJournalCreatesDirOnRecording() throws IOException { 63 | ReactiveJournal journal = new ReactiveJournal(base.toString()); 64 | ReactiveRecorder recorder = journal.createReactiveRecorder(); 65 | 66 | Assert.assertFalse(Files.exists(expectedDir)); 67 | 68 | recorder.record(Flowable.just("foo"), ""); 69 | Assert.assertTrue(Files.exists(expectedDir)); 70 | } 71 | 72 | @Test 73 | public void rxJournalClearsIfOnlyCq4Files() throws IOException { 74 | ReactiveJournal journal = new ReactiveJournal(base.toString()); 75 | 76 | ReactiveRecorder recorder = journal.createReactiveRecorder(); 77 | recorder.record(Flowable.just("foo"), ""); 78 | Assert.assertTrue(Files.exists(expectedDir)); 79 | 80 | //short circuit the test if the dir is unexpected, just in case the clearCache would then be too destructive 81 | Assert.assertEquals(expectedDir.toString(), journal.getDir()); 82 | 83 | journal.clearCache(); 84 | 85 | Assert.assertFalse(Files.exists(expectedDir)); 86 | } 87 | 88 | @Test 89 | public void rxJournalClearsOnlyCq4FilesAndKeepsBaseDir() throws IOException { 90 | Files.createDirectories(expectedDir); 91 | Path tempFile = Files.createTempFile(expectedDir, "rxJournalClearsOnlyCq4", ".txt"); 92 | 93 | ReactiveJournal journal = new ReactiveJournal(base.toString()); 94 | 95 | ReactiveRecorder recorder = journal.createReactiveRecorder(); 96 | recorder.record(Flowable.just("foo"), ""); 97 | 98 | //short circuit the test if the dir is unexpected, just in case the clearCache would then be too destructive 99 | Assert.assertEquals(expectedDir.toString(), journal.getDir()); 100 | 101 | journal.clearCache(); 102 | 103 | Assert.assertTrue(Files.exists(expectedDir)); 104 | Assert.assertTrue(Files.exists(tempFile)); 105 | 106 | long count = Files.walk(expectedDir, 1) 107 | .map(Path::toFile) 108 | .filter(File::isFile) 109 | .count(); 110 | 111 | Assert.assertEquals(1, count); 112 | } 113 | 114 | } 115 | -------------------------------------------------------------------------------- /src/test/java/org/reactivejournal/impl/ReactivePlayerTest.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import io.reactivex.Flowable; 4 | import io.reactivex.flowables.ConnectableFlowable; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | import org.reactivestreams.Subscriber; 8 | import org.reactivestreams.Subscription; 9 | import org.reactivejournal.examples.helloworld.BytesToWordsProcessor; 10 | import org.reactivejournal.examples.helloworld.HelloWorldApp_JounalAsObserver; 11 | import org.reactivejournal.impl.PlayOptions.ReplayRate; 12 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.io.IOException; 17 | import java.util.concurrent.CountDownLatch; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | /** 21 | * Test for ReactiveRecorder 22 | */ 23 | public class ReactivePlayerTest { 24 | private static final Logger LOG = LoggerFactory.getLogger(ReactivePlayerTest.class.getName()); 25 | private static final ReplayRate REPLAY_RATE_STRATEGY = ReplayRate.FAST; 26 | private final String tmpDir = System.getProperty("java.io.tmpdir"); 27 | 28 | 29 | @Test 30 | public void testPlay() throws IOException, InterruptedException { 31 | //Create the rxRecorder but don't delete the cache that has been created. 32 | ReactiveJournal reactiveJournal = new ReactiveJournal("src/test/resources/", ""); 33 | reactiveJournal.writeToFile(tmpDir +"/rctext.txt", true); 34 | 35 | //Get the input from the recorder 36 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 37 | PlayOptions options= new PlayOptions() 38 | .filter(HelloWorldApp_JounalAsObserver.INPUT_FILTER) 39 | .replayRate(REPLAY_RATE_STRATEGY) 40 | .completeAtEndOfFile(false); 41 | ConnectableFlowable observableInput = rxPlayer.play(options).publish(); 42 | 43 | BytesToWordsProcessor bytesToWords = new BytesToWordsProcessor(); 44 | Flowable flowableOutput = bytesToWords.process(observableInput); 45 | 46 | CountDownLatch latch = new CountDownLatch(1); 47 | //Send the output stream to the recorder to be validated against the recorded output 48 | ReactiveValidator reactiveValidator = reactiveJournal.createReactiveValidator(); 49 | reactiveValidator.validate("src/test/resources/", 50 | flowableOutput, HelloWorldApp_JounalAsObserver.OUTPUT_FILTER, new Subscriber() { 51 | @Override 52 | public void onSubscribe(Subscription subscription) { 53 | 54 | } 55 | 56 | @Override 57 | public void onNext(Object o) { 58 | LOG.info(o.toString()); 59 | } 60 | 61 | @Override 62 | public void onError(Throwable throwable) { 63 | LOG.error("Problem in process test [{}]", throwable); 64 | } 65 | 66 | @Override 67 | public void onComplete() { 68 | LOG.info("Summary[" + reactiveValidator.getValidationResult().summaryResult() 69 | + "] items compared[" + reactiveValidator.getValidationResult().summaryItemsCompared() 70 | + "] items valid[" + reactiveValidator.getValidationResult().summaryItemsValid() +"]"); 71 | latch.countDown(); 72 | } 73 | }); 74 | 75 | observableInput.connect(); 76 | boolean completedWithoutTimeout = latch.await(2, TimeUnit.SECONDS); 77 | Assert.assertEquals(ValidationResult.Result.OK, reactiveValidator.getValidationResult().getResult()); 78 | Assert.assertTrue(completedWithoutTimeout); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/test/java/org/reactivejournal/impl/ReactiveRecorderTest.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import io.reactivex.Flowable; 4 | import io.reactivex.flowables.ConnectableFlowable; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | import org.reactivejournal.examples.helloworld.BytesToWordsProcessor; 8 | import org.reactivejournal.examples.helloworld.HelloWorldApp_JournalPlayThrough; 9 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 10 | 11 | import java.io.IOException; 12 | import java.nio.file.Files; 13 | import java.nio.file.Paths; 14 | import java.util.List; 15 | import java.util.stream.Collectors; 16 | 17 | /** 18 | * Test for RxRecorder 19 | */ 20 | public class ReactiveRecorderTest { 21 | private final String tmpDir = System.getProperty("java.io.tmpdir"); 22 | 23 | /* 24 | * Test the record and play functionality as well as the writeToFile. 25 | * 26 | * Record to a journal, play from it into the BytesProcessor and then write the data to a file. 27 | * 28 | * Compare the data in the file with a file produced from a control run. 29 | */ 30 | @Test 31 | public void recorderTest() throws IOException{ 32 | //Flowable used to create the control run. 33 | Flowable observableInput = HelloWorldApp_JournalPlayThrough.observableInput; 34 | 35 | //Create the rxRecorder and delete any previous content by clearing the cache 36 | ReactiveJournal reactiveJournal = new ReactiveJournal(tmpDir +"/playTest"); 37 | reactiveJournal.clearCache(); 38 | 39 | //Pass the input stream into the rxRecorder which will subscribe to it and record all events. 40 | //The subscription will happen on a new thread which will allow this program to continue. 41 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 42 | reactiveRecorder.recordAsync(observableInput, "input"); 43 | 44 | BytesToWordsProcessor bytesToWords = new BytesToWordsProcessor(); 45 | 46 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 47 | PlayOptions options = new PlayOptions().filter("input").playFromNow(true).sameThread(true); 48 | ConnectableFlowable recordedObservable = rxPlayer.play(options).publish(); 49 | //Pass the input Byte stream into the BytesToWordsProcessor class which subscribes to the stream and returns 50 | //a stream of words. 51 | Flowable flowableOutput = bytesToWords.process(recordedObservable); 52 | 53 | //Pass the output stream (of words) into the rxRecorder which will subscribe to it and record all events. 54 | reactiveRecorder.record(flowableOutput, "output"); 55 | 56 | //Only start the recording now because we want to make sure that the BytesToWordsProcessor and the rxRecorder 57 | //are both setup up to receive subscriptions. 58 | recordedObservable.connect(); 59 | reactiveJournal.writeToFile(tmpDir + "/playTest/playTest.txt",true); 60 | 61 | List toBeTested = Files.readAllLines(Paths.get(tmpDir + "/playTest/playTest.txt")); 62 | List controlSet = Files.readAllLines(Paths.get("src/test/resources/playTest.txt")); 63 | 64 | Assert.assertEquals(controlSet.size(), toBeTested.size()); 65 | 66 | //Asert all the values are in both files - they might not be in exactly the same order 67 | 68 | String[] controlSetInput = getFilterLinesFromFiles(controlSet, "input"); 69 | String[] toBeTestedInput= getFilterLinesFromFiles(toBeTested, "input"); 70 | Assert.assertArrayEquals(controlSetInput, toBeTestedInput); 71 | 72 | String[] controlSetOutput = getFilterLinesFromFiles(controlSet, "output"); 73 | String[] toBeTestedOutput= getFilterLinesFromFiles(toBeTested, "output"); 74 | Assert.assertArrayEquals(controlSetOutput, toBeTestedOutput); 75 | 76 | String[] controlSetEOS = getFilterLinesFromFiles(controlSet, "endOfStream"); 77 | String[] toBeTestedEOS= getFilterLinesFromFiles(toBeTested, "endOfStream"); 78 | Assert.assertArrayEquals(controlSetEOS, toBeTestedEOS); 79 | } 80 | 81 | private String[] getFilterLinesFromFiles(List lines, String filter){ 82 | return lines.stream() 83 | .map(this::removeTimeStampAndMessageCount) 84 | .filter(s->s.contains(filter)) 85 | .collect(Collectors.toList()) 86 | .toArray(new String[0]); 87 | } 88 | 89 | private String removeTimeStampAndMessageCount(String line){ 90 | String[] parts = line.split("\\t"); 91 | return parts[3] + "\t" + parts[4]; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/test/java/org/reactivejournal/impl/ReactiveReplayRateTest.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import io.reactivex.BackpressureStrategy; 4 | import io.reactivex.Flowable; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 8 | import org.reactivejournal.util.DSUtil; 9 | 10 | import java.io.IOException; 11 | import java.util.concurrent.atomic.AtomicInteger; 12 | 13 | 14 | /** 15 | * Test designed to test ACTUAL_TIME vs FAST 16 | */ 17 | public class ReactiveReplayRateTest { 18 | @Test 19 | public void replayRateTest() throws IOException { 20 | Flowable errorFlowable = Flowable.create( 21 | e -> { 22 | e.onNext("one"); 23 | DSUtil.sleep(1000); 24 | e.onNext("two"); 25 | e.onComplete(); 26 | }, 27 | BackpressureStrategy.BUFFER 28 | ); 29 | 30 | 31 | ReactiveJournal reactiveJournal = new ReactiveJournal("/tmp/replayRateTest"); 32 | reactiveJournal.clearCache(); 33 | 34 | //Pass the input stream into the rxRecorder which will subscribe to it and record all events. 35 | //The subscription will not be activated on a new thread which will allow this program to continue. 36 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 37 | reactiveRecorder.record(errorFlowable, "replayRateTest"); 38 | 39 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 40 | PlayOptions options = new PlayOptions() 41 | .filter("replayRateTest") 42 | .replayRate(PlayOptions.ReplayRate.ACTUAL_TIME) 43 | .sameThread(true); 44 | Flowable recordedObservable = rxPlayer.play(options); 45 | 46 | AtomicInteger onNext = new AtomicInteger(0); 47 | AtomicInteger onComplete = new AtomicInteger(0); 48 | AtomicInteger onError = new AtomicInteger(0); 49 | 50 | long[] gapBetweenOneAndTwo = new long[1]; 51 | long[] startTime = new long[1]; 52 | 53 | recordedObservable.subscribe(i -> { 54 | String s = (String) i; 55 | if (s.equals("one")) { 56 | startTime[0] = System.currentTimeMillis(); 57 | onNext.incrementAndGet(); 58 | } 59 | if (s.equals("two")) { 60 | gapBetweenOneAndTwo[0] = System.currentTimeMillis() - startTime[0]; 61 | onNext.incrementAndGet(); 62 | } 63 | }, 64 | e -> { 65 | onError.incrementAndGet(); 66 | }, 67 | () -> onComplete.incrementAndGet()); 68 | 69 | Assert.assertEquals(2, onNext.get()); 70 | Assert.assertEquals(0, onError.get()); 71 | Assert.assertEquals(1, onComplete.get()); 72 | System.out.println("Gap with actual was " + gapBetweenOneAndTwo[0]); 73 | Assert.assertTrue(gapBetweenOneAndTwo[0] > 900); 74 | 75 | 76 | rxPlayer = new RxJavaPlayer(reactiveJournal); 77 | options = new PlayOptions().filter("replayRateTest") 78 | .replayRate(PlayOptions.ReplayRate.FAST) 79 | .sameThread(true); 80 | recordedObservable = rxPlayer.play(options); 81 | 82 | AtomicInteger onNextFast = new AtomicInteger(0); 83 | AtomicInteger onCompleteFast = new AtomicInteger(0); 84 | AtomicInteger onErrorFast = new AtomicInteger(0); 85 | 86 | long[] gapBetweenOneAndTwoFast = new long[1]; 87 | long[] startTimeFast = new long[1]; 88 | 89 | recordedObservable.subscribe(i -> { 90 | String s = (String) i; 91 | 92 | if (s.equals("one")) { 93 | startTimeFast[0] = System.currentTimeMillis(); 94 | onNextFast.incrementAndGet(); 95 | } 96 | if (s.equals("two")) { 97 | gapBetweenOneAndTwoFast[0] = System.currentTimeMillis() - startTimeFast[0]; 98 | onNextFast.incrementAndGet(); 99 | } 100 | }, 101 | e -> { 102 | onErrorFast.incrementAndGet(); 103 | }, 104 | () -> onCompleteFast.incrementAndGet()); 105 | 106 | Assert.assertEquals(2, onNextFast.get()); 107 | Assert.assertEquals(0, onErrorFast.get()); 108 | Assert.assertEquals(1, onCompleteFast.get()); 109 | System.out.println("Gap with fast was " + gapBetweenOneAndTwoFast[0]); 110 | Assert.assertTrue(gapBetweenOneAndTwoFast[0] < 2); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/test/java/org/reactivejournal/impl/ReactiveRequestTest.java: -------------------------------------------------------------------------------- 1 | package org.reactivejournal.impl; 2 | 3 | import io.reactivex.BackpressureStrategy; 4 | import io.reactivex.Flowable; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | import org.reactivestreams.Subscriber; 8 | import org.reactivestreams.Subscription; 9 | import org.reactivejournal.impl.rxjava.RxJavaPlayer; 10 | 11 | import java.io.IOException; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | import java.util.concurrent.CountDownLatch; 15 | import java.util.concurrent.TimeUnit; 16 | import java.util.concurrent.atomic.AtomicInteger; 17 | 18 | 19 | /** 20 | * Test cases for request 21 | */ 22 | public class ReactiveRequestTest { 23 | @Test 24 | public void testRequest() throws IOException{ 25 | test(10, 1,1, true); 26 | test(100, 1000,1, true); 27 | test(10, 1,3, true); 28 | test(10, 1,1, false); 29 | test(100, 1000,1, false); 30 | test(10, 1,3, false); 31 | } 32 | 33 | private void test(int messageCount, int initialRequest, int onNextRequest, boolean sameThread) throws IOException { 34 | CountDownLatch latch = new CountDownLatch(1); 35 | 36 | Flowable errorFlowable = Flowable.create( 37 | e -> { 38 | for (int i = 0; i < messageCount; i++) { 39 | e.onNext("" + i); 40 | } 41 | e.onComplete(); 42 | }, 43 | BackpressureStrategy.BUFFER 44 | ); 45 | 46 | 47 | ReactiveJournal reactiveJournal = new ReactiveJournal("/tmp/testRequest"); 48 | reactiveJournal.clearCache(); 49 | 50 | //Pass the input stream into the reactiveRecorder which will subscribe to it and record all events. 51 | //The subscription will not be activated on a new thread which will allow this program to continue. 52 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 53 | reactiveRecorder.recordAsync(errorFlowable, "request"); 54 | 55 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 56 | PlayOptions options = new PlayOptions().filter("request").sameThread(sameThread); 57 | Flowable recordedObservable = rxPlayer.play(options); 58 | 59 | AtomicInteger onNext = new AtomicInteger(0); 60 | AtomicInteger onComplete = new AtomicInteger(0); 61 | AtomicInteger onError = new AtomicInteger(0); 62 | 63 | List results = new ArrayList(); 64 | //Pass the output stream (of words) into the reactiveRecorder which will subscribe to it and record all events. 65 | recordedObservable.subscribe(new Subscriber() { 66 | private Subscription subscription; 67 | 68 | @Override 69 | public void onSubscribe(Subscription subscription) { 70 | this.subscription = subscription; 71 | subscription.request(initialRequest); 72 | } 73 | 74 | @Override 75 | public void onNext(Object o) { 76 | onNext.incrementAndGet(); 77 | results.add(o); 78 | subscription.request(onNextRequest); 79 | } 80 | 81 | @Override 82 | public void onError(Throwable throwable) { 83 | onError.incrementAndGet(); 84 | } 85 | 86 | @Override 87 | public void onComplete() { 88 | onComplete.incrementAndGet(); 89 | latch.countDown(); 90 | } 91 | 92 | }); 93 | 94 | 95 | try { 96 | latch.await(3000, TimeUnit.SECONDS); 97 | } catch (InterruptedException e) { 98 | e.printStackTrace(); 99 | } 100 | 101 | Assert.assertEquals(messageCount, onNext.get()); 102 | Assert.assertEquals(0, onError.get()); 103 | Assert.assertEquals(1, onComplete.get()); 104 | 105 | for(int i=0; i errorFlowable = Flowable.create( 126 | e -> { 127 | for (int i = 0; i < messageCount; i++) { 128 | e.onNext("" + i); 129 | } 130 | e.onComplete(); 131 | }, 132 | BackpressureStrategy.BUFFER 133 | ); 134 | 135 | 136 | ReactiveJournal reactiveJournal = new ReactiveJournal("/tmp/testRequest"); 137 | reactiveJournal.clearCache(); 138 | 139 | //Pass the input stream into the reactiveRecorder which will subscribe to it and record all events. 140 | //The subscription will not be activated on a new thread which will allow this program to continue. 141 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 142 | reactiveRecorder.recordAsync(errorFlowable, "request"); 143 | 144 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 145 | PlayOptions options = new PlayOptions().filter("request").sameThread(sameThread); 146 | Flowable recordedObservable = rxPlayer.play(options); 147 | 148 | AtomicInteger onNext = new AtomicInteger(0); 149 | AtomicInteger onComplete = new AtomicInteger(0); 150 | AtomicInteger onError = new AtomicInteger(0); 151 | 152 | List results = new ArrayList(); 153 | //Pass the output stream (of words) into the reactiveRecorder which will subscribe to it and record all events. 154 | recordedObservable.subscribe(new Subscriber() { 155 | private Subscription subscription; 156 | 157 | @Override 158 | public void onSubscribe(Subscription subscription) { 159 | this.subscription = subscription; 160 | subscription.request(initialRequest); 161 | } 162 | 163 | @Override 164 | public void onNext(Object o) { 165 | onNext.incrementAndGet(); 166 | results.add(o); 167 | subscription.request(onNextRequest); 168 | subscription.cancel(); 169 | } 170 | 171 | @Override 172 | public void onError(Throwable throwable) { 173 | onError.incrementAndGet(); 174 | } 175 | 176 | @Override 177 | public void onComplete() { 178 | onComplete.incrementAndGet(); 179 | latch.countDown(); 180 | } 181 | 182 | }); 183 | 184 | 185 | 186 | try { 187 | boolean await = latch.await(500, TimeUnit.MILLISECONDS); 188 | Assert.assertFalse(await); 189 | } catch (InterruptedException e) { 190 | e.printStackTrace(); 191 | } 192 | 193 | Assert.assertEquals(1, onNext.get()); 194 | Assert.assertEquals(0, onError.get()); 195 | Assert.assertEquals(0, onComplete.get()); 196 | 197 | for(int i=0; i<1; i++){ 198 | Assert.assertEquals(""+i, results.get(i)); 199 | } 200 | 201 | } 202 | 203 | @Test 204 | public void testFastPath() throws IOException { 205 | testFastPath(true); 206 | testFastPath(false); 207 | } 208 | 209 | public void testFastPath(boolean sameThread) throws IOException{ 210 | int messageCount = 10; 211 | int initialRequest = 1; 212 | int onNextRequest = 1; 213 | 214 | CountDownLatch latch = new CountDownLatch(1); 215 | 216 | Flowable errorFlowable = Flowable.create( 217 | e -> { 218 | for (int i = 0; i < messageCount; i++) { 219 | e.onNext("" + i); 220 | } 221 | e.onComplete(); 222 | }, 223 | BackpressureStrategy.BUFFER 224 | ); 225 | 226 | 227 | ReactiveJournal reactiveJournal = new ReactiveJournal("/tmp/testRequest"); 228 | reactiveJournal.clearCache(); 229 | 230 | //Pass the input stream into the reactiveRecorder which will subscribe to it and record all events. 231 | //The subscription will not be activated on a new thread which will allow this program to continue. 232 | ReactiveRecorder reactiveRecorder = reactiveJournal.createReactiveRecorder(); 233 | reactiveRecorder.recordAsync(errorFlowable, "request"); 234 | 235 | RxJavaPlayer rxPlayer = new RxJavaPlayer(reactiveJournal); 236 | PlayOptions options = new PlayOptions().filter("request").sameThread(sameThread); 237 | Flowable recordedObservable = rxPlayer.play(options); 238 | 239 | AtomicInteger onNext = new AtomicInteger(0); 240 | AtomicInteger onComplete = new AtomicInteger(0); 241 | AtomicInteger onError = new AtomicInteger(0); 242 | 243 | List results = new ArrayList(); 244 | //Pass the output stream (of words) into the reactiveRecorder which will subscribe to it and record all events. 245 | recordedObservable.subscribe(new Subscriber() { 246 | private Subscription subscription; 247 | 248 | @Override 249 | public void onSubscribe(Subscription subscription) { 250 | this.subscription = subscription; 251 | subscription.request(initialRequest); 252 | } 253 | 254 | @Override 255 | public void onNext(Object o) { 256 | onNext.incrementAndGet(); 257 | results.add(o); 258 | subscription.request(Long.MAX_VALUE); 259 | subscription.cancel(); 260 | } 261 | 262 | @Override 263 | public void onError(Throwable throwable) { 264 | onError.incrementAndGet(); 265 | } 266 | 267 | @Override 268 | public void onComplete() { 269 | onComplete.incrementAndGet(); 270 | latch.countDown(); 271 | System.out.println("COMPLETE"); 272 | } 273 | 274 | }); 275 | 276 | 277 | 278 | try { 279 | boolean await = latch.await(1000, TimeUnit.MILLISECONDS); 280 | Assert.assertFalse(await); 281 | } catch (InterruptedException e) { 282 | e.printStackTrace(); 283 | } 284 | 285 | Assert.assertEquals(1, onNext.get()); 286 | Assert.assertEquals(0, onError.get()); 287 | Assert.assertEquals(0, onComplete.get()); 288 | 289 | for(int i=0; i<1; i++){ 290 | Assert.assertEquals(""+i, results.get(i)); 291 | } 292 | 293 | } 294 | } 295 | -------------------------------------------------------------------------------- /src/test/resources/20170612.cq4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danielshaya/reactivejournal/cc441731729cba62457481d94489ca97b8966545/src/test/resources/20170612.cq4 -------------------------------------------------------------------------------- /src/test/resources/playTest.txt: -------------------------------------------------------------------------------- 1 | VALID 1 2017-05-11T11:01:20.135 input 72 2 | VALID 2 2017-05-11T11:01:20.137 input 101 3 | VALID 3 2017-05-11T11:01:20.138 input 108 4 | VALID 4 2017-05-11T11:01:20.139 input 108 5 | VALID 5 2017-05-11T11:01:20.143 input 111 6 | VALID 6 2017-05-11T11:01:20.144 input 32 7 | VALID 7 2017-05-11T11:01:20.146 input 87 8 | VALID 8 2017-05-11T11:01:20.148 input 111 9 | VALID 9 2017-05-11T11:01:20.150 input 114 10 | VALID 10 2017-05-11T11:01:20.151 input 108 11 | VALID 11 2017-05-11T11:01:20.152 input 100 12 | VALID 12 2017-05-11T11:01:20.154 input 32 13 | COMPLETE 13 2017-05-11T11:01:20.154 input EndOfStream{} 14 | VALID 14 2017-05-11T11:01:20.173 output Hello 15 | VALID 15 2017-05-11T11:01:20.184 output World 16 | COMPLETE 16 2017-05-11T11:01:20.184 output EndOfStream{} 17 | --------------------------------------------------------------------------------