├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── OSSMETADATA ├── README.md ├── build.gradle ├── codequality ├── checkstyle.xml ├── netflix-intellij-code-style.xml └── pmd.xml ├── core ├── build.gradle └── src │ └── main │ └── java │ └── com │ └── netflix │ └── mantis │ └── examples │ ├── config │ └── StageConfigs.java │ └── core │ ├── ObservableQueue.java │ └── WordCountPair.java ├── gradle ├── check.gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── groupby-sample ├── build.gradle └── src │ └── main │ ├── java │ └── com │ │ └── netflix │ │ └── mantis │ │ └── samples │ │ ├── RequestAggregationJob.java │ │ ├── proto │ │ ├── AggregationReport.java │ │ ├── RequestAggregation.java │ │ └── RequestEvent.java │ │ ├── source │ │ └── RandomRequestSource.java │ │ └── stage │ │ ├── AggregationStage.java │ │ ├── CollectStage.java │ │ └── GroupByStage.java │ └── resources │ ├── META-INF │ └── services │ │ └── io.mantisrx.runtime.MantisJobProvider │ └── log4j.properties ├── installViaTravis.sh ├── jobconnector-sample ├── build.gradle └── src │ └── main │ ├── java │ └── com │ │ └── netflix │ │ └── mantis │ │ └── samples │ │ ├── JobConnectorJob.java │ │ └── stage │ │ └── EchoStage.java │ └── resources │ ├── META-INF │ └── services │ │ └── io.mantisrx.runtime.MantisJobProvider │ └── log4j.properties ├── mantis-publish-sample ├── Dockerfile ├── README.md ├── build.gradle ├── buildDockerImage.sh └── src │ └── main │ ├── java │ └── com │ │ └── netflix │ │ └── mantis │ │ └── examples │ │ └── mantispublishsample │ │ ├── Application.java │ │ ├── BasicModule.java │ │ ├── DataGenerator.java │ │ ├── IDataGenerator.java │ │ ├── IDataPublisher.java │ │ ├── SampleDataPublisher.java │ │ └── proto │ │ └── RequestEvent.java │ └── resources │ ├── application.properties │ └── log4j.properties ├── mantis-publish-web-sample ├── Dockerfile ├── README.md ├── build.gradle ├── buildDockerImage.sh └── src │ └── main │ ├── java │ └── com │ │ └── netflix │ │ └── mantis │ │ └── examples │ │ └── mantispublishsample │ │ └── web │ │ ├── config │ │ └── DefaultGuiceServletConfig.java │ │ ├── filter │ │ └── CaptureRequestEventFilter.java │ │ ├── service │ │ ├── MyService.java │ │ └── MyServiceImpl.java │ │ └── servlet │ │ └── HelloServlet.java │ ├── resources │ ├── application.properties │ └── log4j.properties │ └── webapp │ ├── WEB-INF │ └── web.xml │ ├── index.html │ └── response.jsp ├── settings.gradle ├── sine-function ├── build.gradle └── src │ └── main │ ├── java │ └── io │ │ └── mantisrx │ │ └── mantis │ │ └── examples │ │ └── sinefunction │ │ ├── SineFunctionJob.java │ │ ├── core │ │ └── Point.java │ │ └── stages │ │ └── SinePointGeneratorStage.java │ └── resources │ ├── META-INF │ └── services │ │ └── io.mantisrx.runtime.MantisJobProvider │ └── log4j.properties ├── synthetic-sourcejob ├── build.gradle └── src │ └── main │ ├── java │ └── io │ │ └── mantisrx │ │ └── sourcejob │ │ └── synthetic │ │ ├── SyntheticSourceJob.java │ │ ├── core │ │ ├── MQL.java │ │ ├── MQLQueryManager.java │ │ └── TaggedData.java │ │ ├── proto │ │ └── RequestEvent.java │ │ ├── sink │ │ ├── QueryRefCountMap.java │ │ ├── QueryRequestPostProcessor.java │ │ ├── QueryRequestPreProcessor.java │ │ ├── TaggedDataSourceSink.java │ │ └── TaggedEventFilter.java │ │ ├── source │ │ └── SyntheticSource.java │ │ └── stage │ │ └── TaggingStage.java │ └── resources │ ├── META-INF │ └── services │ │ └── io.mantisrx.runtime.MantisJobProvider │ └── log4j.properties ├── twitter-sample ├── build.gradle └── src │ └── main │ ├── java │ └── com │ │ └── netflix │ │ └── mantis │ │ └── examples │ │ ├── config │ │ └── StageConfigs.java │ │ ├── core │ │ ├── ObservableQueue.java │ │ └── WordCountPair.java │ │ └── wordcount │ │ ├── TwitterJob.java │ │ └── sources │ │ └── TwitterSource.java │ └── resources │ ├── META-INF │ └── services │ │ └── io.mantisrx.runtime.MantisJobProvider │ └── log4j.properties └── wordcount ├── build.gradle └── src └── main ├── java └── com │ └── netflix │ └── mantis │ └── examples │ └── wordcount │ ├── WordCountJob.java │ └── sources │ └── IlliadSource.java └── resources ├── META-INF └── services │ └── io.mantisrx.runtime.MantisJobProvider ├── illiad.txt └── log4j.properties /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @codyrioux @neerajrj @nickmahilani @jeffchao @piygoyal @skalidindi 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Context 2 | 3 | Explain symptoms and other details for the issue. 4 | 5 | ### Steps to reproduce 6 | 7 | Explain the steps to reliably reproduce the issue. 8 | 9 | ### Expected behavior 10 | 11 | Explain what you think should happen. 12 | 13 | ### Actual Behavior 14 | 15 | Explain what actually happens instead. 16 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Context 2 | 3 | Explain context and other details for this pull request. 4 | 5 | ### Checklist 6 | 7 | - [ ] `./gradlew build` compiles code correctly 8 | - [ ] Added new tests where applicable 9 | - [ ] `./gradlew test` passes all tests 10 | - [ ] Extended README or added javadocs where applicable 11 | - [ ] Added copyright headers for new files from `CONTRIBUTING.md` 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/vim,java,linux,macos,gradle,intellij+all 3 | # Edit at https://www.gitignore.io/?templates=vim,java,linux,macos,gradle,intellij+all 4 | 5 | ### Intellij+all ### 6 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 7 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 8 | 9 | # User-specific stuff 10 | .idea/**/workspace.xml 11 | .idea/**/tasks.xml 12 | .idea/**/usage.statistics.xml 13 | .idea/**/dictionaries 14 | .idea/**/shelf 15 | 16 | # Generated files 17 | .idea/**/contentModel.xml 18 | 19 | # Sensitive or high-churn files 20 | .idea/**/dataSources/ 21 | .idea/**/dataSources.ids 22 | .idea/**/dataSources.local.xml 23 | .idea/**/sqlDataSources.xml 24 | .idea/**/dynamic.xml 25 | .idea/**/uiDesigner.xml 26 | .idea/**/dbnavigator.xml 27 | 28 | # Gradle 29 | .idea/**/gradle.xml 30 | .idea/**/libraries 31 | 32 | # Gradle and Maven with auto-import 33 | # When using Gradle or Maven with auto-import, you should exclude module files, 34 | # since they will be recreated, and may cause churn. Uncomment if using 35 | # auto-import. 36 | # .idea/modules.xml 37 | # .idea/*.iml 38 | # .idea/modules 39 | 40 | # CMake 41 | cmake-build-*/ 42 | 43 | # Mongo Explorer plugin 44 | .idea/**/mongoSettings.xml 45 | 46 | # File-based project format 47 | *.iws 48 | 49 | # IntelliJ 50 | out/ 51 | 52 | # mpeltonen/sbt-idea plugin 53 | .idea_modules/ 54 | 55 | # JIRA plugin 56 | atlassian-ide-plugin.xml 57 | 58 | # Cursive Clojure plugin 59 | .idea/replstate.xml 60 | 61 | # Crashlytics plugin (for Android Studio and IntelliJ) 62 | com_crashlytics_export_strings.xml 63 | crashlytics.properties 64 | crashlytics-build.properties 65 | fabric.properties 66 | 67 | # Editor-based Rest Client 68 | .idea/httpRequests 69 | 70 | # Android studio 3.1+ serialized cache file 71 | .idea/caches/build_file_checksums.ser 72 | 73 | # JetBrains templates 74 | **___jb_tmp___ 75 | 76 | ### Intellij+all Patch ### 77 | # Ignores the whole .idea folder and all .iml files 78 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 79 | 80 | .idea/ 81 | 82 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 83 | 84 | *.iml 85 | modules.xml 86 | .idea/misc.xml 87 | *.ipr 88 | 89 | # Sonarlint plugin 90 | .idea/sonarlint 91 | 92 | ### Java ### 93 | # Compiled class file 94 | *.class 95 | 96 | # Log file 97 | *.log 98 | 99 | # BlueJ files 100 | *.ctxt 101 | 102 | # Mobile Tools for Java (J2ME) 103 | .mtj.tmp/ 104 | 105 | # Package Files # 106 | *.jar 107 | *.war 108 | *.nar 109 | *.ear 110 | *.zip 111 | *.tar.gz 112 | *.rar 113 | 114 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 115 | hs_err_pid* 116 | 117 | ### Linux ### 118 | *~ 119 | 120 | # temporary files which can be created if a process still has a handle open of a deleted file 121 | .fuse_hidden* 122 | 123 | # KDE directory preferences 124 | .directory 125 | 126 | # Linux trash folder which might appear on any partition or disk 127 | .Trash-* 128 | 129 | # .nfs files are created when an open file is removed but is still being accessed 130 | .nfs* 131 | 132 | ### macOS ### 133 | # General 134 | .DS_Store 135 | .AppleDouble 136 | .LSOverride 137 | 138 | # Icon must end with two \r 139 | Icon 140 | 141 | # Thumbnails 142 | ._* 143 | 144 | # Files that might appear in the root of a volume 145 | .DocumentRevisions-V100 146 | .fseventsd 147 | .Spotlight-V100 148 | .TemporaryItems 149 | .Trashes 150 | .VolumeIcon.icns 151 | .com.apple.timemachine.donotpresent 152 | 153 | # Directories potentially created on remote AFP share 154 | .AppleDB 155 | .AppleDesktop 156 | Network Trash Folder 157 | Temporary Items 158 | .apdisk 159 | 160 | ### Vim ### 161 | # Swap 162 | [._]*.s[a-v][a-z] 163 | [._]*.sw[a-p] 164 | [._]s[a-rt-v][a-z] 165 | [._]ss[a-gi-z] 166 | [._]sw[a-p] 167 | 168 | # Session 169 | Session.vim 170 | 171 | # Temporary 172 | .netrwhist 173 | # Auto-generated tag files 174 | tags 175 | # Persistent undo 176 | [._]*.un~ 177 | 178 | ### Gradle ### 179 | .gradle 180 | build/ 181 | 182 | # Ignore Gradle GUI com.netflix.mantis.examples.config 183 | gradle-app.setting 184 | 185 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 186 | !gradle-wrapper.jar 187 | 188 | # Cache of project 189 | .gradletasknamecache 190 | 191 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 192 | # gradle/wrapper/gradle-wrapper.properties 193 | 194 | ### Gradle Patch ### 195 | **/build/ 196 | 197 | # End of https://www.gitignore.io/api/vim,java,linux,macos,gradle,intellij+all 198 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: openjdk8 3 | install: "./installViaTravis.sh" 4 | script: ./gradlew -Prelease.travisci=true build --stacktrace 5 | cache: 6 | directories: 7 | - "$HOME/.gradle/caches/" 8 | - "$HOME/.gradle/wrapper/" 9 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Mantis 2 | 3 | If you would like to contribute code you can do so through GitHub by forking the repository and sending a pull request. When submitting code, please make every effort to follow existing conventions and style in order to keep the code as readable as possible. 4 | 5 | ## License 6 | 7 | By contributing your code, you agree to license your contribution under the terms of the [Apache License v2.0](http://www.apache.org/licenses/LICENSE-2.0). Your contributions should also include the following header: 8 | 9 | ``` 10 | /** 11 | * Copyright 2019 Netflix, Inc. 12 | * 13 | * Licensed under the Apache License, Version 2.0 (the "License"); 14 | * you may not use this file except in compliance with the License. 15 | * You may obtain a copy of the License at 16 | * 17 | * http://www.apache.org/licenses/LICENSE-2.0 18 | * 19 | * Unless required by applicable law or agreed to in writing, software 20 | * distributed under the License is distributed on an "AS IS" BASIS, 21 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 22 | * See the License for the specific language governing permissions and 23 | * limitations under the License. 24 | */ 25 | ``` 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/mantis-examples/b6bd7717ed4ea66f90e371603806cd7d4ef333aa/NOTICE -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=archived 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mantis-examples 2 | 3 | DEPRECATED - moved to https://github.com/netflix/mantis. 4 | 5 | [![Build Status](https://img.shields.io/travis/com/Netflix/mantis-examples.svg)](https://travis-ci.com/Netflix/mantis-examples) 6 | [![OSS Lifecycle](https://img.shields.io/osslifecycle/Netflix/mantis-examples.svg)](https://github.com/Netflix/mantis-examples) 7 | [![License](https://img.shields.io/github/license/Netflix/mantis-examples.svg)](https://www.apache.org/licenses/LICENSE-2.0) 8 | 9 | A list of simple Mantis jobs demonstrating various capabilities. 10 | 11 | ## sine-function 12 | A simple single stage job that generates x and y co-ordinates for a sine wave. 13 | 14 | ## twitter-sample 15 | Processes a twitter stream and performs a word count over a hopping window 16 | 17 | ## groupby-sample 18 | A multi-stage job that generates synthetic request event data, groups the data 19 | by URI and then calculates counts per URI over a rolling window. 20 | 21 | ## synthetic-sourcejob 22 | A sample source job that serves a stream of request event data and allows the consumers 23 | to query against it using the MQL language. 24 | 25 | ## jobconnector-sample 26 | A simple example that demonstrates how to pipe the output of another job 27 | into your job. 28 | 29 | ## mantis-publish-sample 30 | An example of using the mantis-publish library to send events to Mantis. 31 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | buildscript { 18 | repositories { 19 | jcenter() 20 | maven { 21 | url "https://plugins.gradle.org/m2/" 22 | } 23 | } 24 | 25 | dependencies { 26 | classpath 'com.netflix.nebula:gradle-netflixoss-project-plugin:5.1.1' 27 | classpath 'io.mantisrx:mantis-gradle-plugin:1.2.+' 28 | } 29 | } 30 | 31 | ext { 32 | junitVersion = '4.10' 33 | } 34 | 35 | allprojects { 36 | apply plugin: 'nebula.netflixoss' 37 | apply plugin: 'idea' 38 | } 39 | 40 | subprojects { 41 | apply plugin: 'java-library' 42 | apply plugin: 'checkstyle' 43 | apply plugin: 'pmd' 44 | apply plugin: 'mantis' 45 | 46 | sourceCompatibility = JavaVersion.VERSION_1_8 47 | targetCompatibility = JavaVersion.VERSION_1_8 48 | 49 | group = 'io.mantisrx' 50 | 51 | if (project.hasProperty('useMavenLocal')) { 52 | repositories { 53 | mavenLocal() 54 | } 55 | } 56 | 57 | repositories { 58 | jcenter() 59 | } 60 | 61 | dependencies { 62 | testCompile 'junit:junit-dep:4.10' 63 | } 64 | 65 | tasks.withType(Javadoc).all { 66 | enabled = false 67 | } 68 | 69 | license { 70 | ignoreFailures = false 71 | } 72 | } 73 | 74 | apply from: file('gradle/check.gradle') 75 | -------------------------------------------------------------------------------- /codequality/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 21 | 22 | 23 | 24 | 25 | 26 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 158 | 159 | 160 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | -------------------------------------------------------------------------------- /codequality/netflix-intellij-code-style.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 23 | 28 | 41 | -------------------------------------------------------------------------------- /codequality/pmd.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 23 | 24 | Exclude noisy rules. 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /core/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | 3 | dependencies { 4 | compile 'io.mantisrx:mantis-runtime:1.2.+' 5 | compile 'org.slf4j:slf4j-api' 6 | compile 'org.slf4j:slf4j-log4j12' 7 | compileOnly "org.projectlombok:lombok:1.16.16" 8 | } 9 | -------------------------------------------------------------------------------- /core/src/main/java/com/netflix/mantis/examples/config/StageConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.config; 18 | 19 | import java.util.Map; 20 | 21 | import io.mantisrx.common.codec.Codecs; 22 | import io.mantisrx.runtime.KeyToScalar; 23 | import io.mantisrx.runtime.ScalarToKey; 24 | import io.mantisrx.runtime.ScalarToScalar; 25 | import io.mantisrx.runtime.codec.JacksonCodecs; 26 | 27 | 28 | public class StageConfigs { 29 | 30 | public static ScalarToScalar.Config scalarToScalarConfig() { 31 | return new ScalarToScalar.Config() 32 | .codec(Codecs.string()); 33 | } 34 | 35 | public static KeyToScalar.Config, String> keyToScalarConfig() { 36 | return new KeyToScalar.Config, String>() 37 | .description("sum events ") 38 | .keyExpireTimeSeconds(10) 39 | .codec(Codecs.string()); 40 | } 41 | 42 | public static ScalarToKey.Config> scalarToKeyConfig() { 43 | return new ScalarToKey.Config>() 44 | .description("Group event data by ip") 45 | .concurrentInput() 46 | .keyExpireTimeSeconds(1) 47 | .codec(JacksonCodecs.mapStringObject()); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /core/src/main/java/com/netflix/mantis/examples/core/ObservableQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.core; 18 | 19 | import java.io.Closeable; 20 | import java.io.IOException; 21 | import java.util.Collection; 22 | import java.util.Collections; 23 | import java.util.Iterator; 24 | import java.util.NoSuchElementException; 25 | import java.util.concurrent.BlockingQueue; 26 | import java.util.concurrent.TimeUnit; 27 | 28 | import rx.Observable; 29 | import rx.subjects.PublishSubject; 30 | import rx.subjects.Subject; 31 | 32 | 33 | /** 34 | * An Observable that acts as a blocking queue. It is backed by a Subject 35 | * 36 | * @param 37 | */ 38 | public class ObservableQueue implements BlockingQueue, Closeable { 39 | 40 | private final Subject subject = PublishSubject.create().toSerialized(); 41 | 42 | public Observable observe() { 43 | return subject; 44 | } 45 | 46 | @Override 47 | public boolean add(T t) { 48 | return offer(t); 49 | } 50 | 51 | @Override 52 | public boolean offer(T t) { 53 | subject.onNext(t); 54 | return true; 55 | } 56 | 57 | @Override 58 | public void close() throws IOException { 59 | subject.onCompleted(); 60 | } 61 | 62 | @Override 63 | public T remove() { 64 | return noSuchElement(); 65 | } 66 | 67 | @Override 68 | public T poll() { 69 | return null; 70 | } 71 | 72 | @Override 73 | public T element() { 74 | return noSuchElement(); 75 | } 76 | 77 | private T noSuchElement() { 78 | throw new NoSuchElementException(); 79 | } 80 | 81 | @Override 82 | public T peek() { 83 | return null; 84 | } 85 | 86 | @Override 87 | public void put(T t) throws InterruptedException { 88 | offer(t); 89 | } 90 | 91 | @Override 92 | public boolean offer(T t, long timeout, TimeUnit unit) throws InterruptedException { 93 | return offer(t); 94 | } 95 | 96 | @Override 97 | public T take() throws InterruptedException { 98 | throw new UnsupportedOperationException("Use observe() instead"); 99 | } 100 | 101 | @Override 102 | public T poll(long timeout, TimeUnit unit) throws InterruptedException { 103 | return null; 104 | } 105 | 106 | @Override 107 | public int remainingCapacity() { 108 | return 0; 109 | } 110 | 111 | @Override 112 | public boolean remove(Object o) { 113 | return false; 114 | } 115 | 116 | @Override 117 | public boolean containsAll(Collection c) { 118 | return false; 119 | } 120 | 121 | @Override 122 | public boolean addAll(Collection c) { 123 | c.forEach(this::offer); 124 | return true; 125 | } 126 | 127 | @Override 128 | public boolean removeAll(Collection c) { 129 | return false; 130 | } 131 | 132 | @Override 133 | public boolean retainAll(Collection c) { 134 | return false; 135 | } 136 | 137 | @Override 138 | public void clear() { 139 | } 140 | 141 | @Override 142 | public int size() { 143 | return 0; 144 | } 145 | 146 | @Override 147 | public boolean isEmpty() { 148 | return true; 149 | } 150 | 151 | @Override 152 | public boolean contains(Object o) { 153 | return false; 154 | } 155 | 156 | @Override 157 | public Iterator iterator() { 158 | return Collections.emptyIterator(); 159 | } 160 | 161 | @Override 162 | public Object[] toArray() { 163 | return new Object[0]; 164 | } 165 | 166 | @Override 167 | public T[] toArray(T[] a) { 168 | return a; 169 | } 170 | 171 | @Override 172 | public int drainTo(Collection c) { 173 | return 0; 174 | } 175 | 176 | @Override 177 | public int drainTo(Collection c, int maxElements) { 178 | return 0; 179 | } 180 | 181 | } -------------------------------------------------------------------------------- /core/src/main/java/com/netflix/mantis/examples/core/WordCountPair.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.core; 18 | 19 | import lombok.Data; 20 | 21 | 22 | /** 23 | * A simple class that holds a word and a count of how many times it has occurred. 24 | */ 25 | @Data 26 | public class WordCountPair { 27 | 28 | private final String word; 29 | private final int count; 30 | } 31 | -------------------------------------------------------------------------------- /gradle/check.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | subprojects { 18 | checkstyle { 19 | toolVersion = '8.14' 20 | // TODO: Don't ignore failures. 21 | ignoreFailures = true 22 | configFile = rootProject.file('codequality/checkstyle.xml') 23 | sourceSets = [sourceSets.main] 24 | } 25 | 26 | pmd { 27 | toolVersion = '6.9.0' 28 | // TODO: Don't ignore failures. 29 | ignoreFailures = true 30 | sourceSets = [sourceSets.main] 31 | ruleSets = [] 32 | ruleSetFiles = rootProject.files("codequality/pmd.xml") 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/mantis-examples/b6bd7717ed4ea66f90e371603806cd7d4ef333aa/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Fri Oct 04 20:49:31 PDT 2019 2 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.3-all.zip 3 | distributionBase=GRADLE_USER_HOME 4 | distributionPath=wrapper/dists 5 | zipStorePath=wrapper/dists 6 | zipStoreBase=GRADLE_USER_HOME 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /groupby-sample/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | 3 | configurations.all { 4 | resolutionStrategy { 5 | force "com.google.guava:guava:18.0" 6 | 7 | } 8 | } 9 | 10 | task execute(type:JavaExec) { 11 | main = "com.netflix.mantis.samples.RequestAggregationJob" 12 | classpath = sourceSets.main.runtimeClasspath 13 | } 14 | 15 | dependencies { 16 | 17 | compile 'net.andreinc.mockneat:mockneat:0.3.7' 18 | compile 'io.mantisrx:mantis-runtime:1.2.+' 19 | compile 'com.fasterxml.jackson.core:jackson-core:2.11.1' 20 | compile 'com.fasterxml.jackson.core:jackson-databind:2.11.1' 21 | 22 | compile 'org.slf4j:slf4j-api' 23 | compile 'org.slf4j:slf4j-log4j12' 24 | 25 | compileOnly "org.projectlombok:lombok:1.16.16" 26 | } 27 | 28 | 29 | -------------------------------------------------------------------------------- /groupby-sample/src/main/java/com/netflix/mantis/samples/RequestAggregationJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.samples; 18 | 19 | import com.netflix.mantis.samples.source.RandomRequestSource; 20 | import com.netflix.mantis.samples.stage.AggregationStage; 21 | import com.netflix.mantis.samples.stage.CollectStage; 22 | import com.netflix.mantis.samples.stage.GroupByStage; 23 | import io.mantisrx.runtime.Job; 24 | import io.mantisrx.runtime.MantisJob; 25 | import io.mantisrx.runtime.MantisJobProvider; 26 | import io.mantisrx.runtime.Metadata; 27 | import io.mantisrx.runtime.executor.LocalJobExecutorNetworked; 28 | import io.mantisrx.runtime.sink.Sinks; 29 | import lombok.extern.slf4j.Slf4j; 30 | 31 | 32 | /** 33 | * This sample demonstrates the use of a multi-stage job in Mantis. Multi-stage jobs are useful when a single 34 | * container is incapable of processing the entire stream of events. 35 | * Each stage represents one of these types of 36 | * computations Scalar->Scalar, Scalar->Group, Group->Scalar, Group->Group. 37 | * 38 | * At deploy time the user can configure the number workers for each stage and the resource requirements for each worker. 39 | * This sample has 3 stages 40 | * 1. {@link GroupByStage} Receives the raw events, groups them by their category and sends it to the workers of stage 2 in such a way 41 | * that all events for a particular group will land on the exact same worker of stage 2. 42 | * 2. {@link AggregationStage} Receives events tagged by their group from the previous stage. Windows over them and 43 | * sums up the counts of each group it has seen. 44 | * 3. {@link CollectStage} Recieves the aggregates generated by the previous stage, collects them over a window and 45 | * generates a consolidated report which is sent to the default Server Sent Event (SSE) sink. 46 | * 47 | * Run this sample by executing the main method of this class. Then look for the SSE port where the output of this job 48 | * will be available for streaming. E.g Serving modern HTTP SSE server sink on port: 8299 49 | * via command line do ../gradlew execute 50 | */ 51 | 52 | @Slf4j 53 | public class RequestAggregationJob extends MantisJobProvider { 54 | 55 | @Override 56 | public Job getJobInstance() { 57 | 58 | return MantisJob 59 | // Stream Request Events from our random data generator source 60 | .source(new RandomRequestSource()) 61 | 62 | // Groups requests by path 63 | .stage(new GroupByStage(), GroupByStage.config()) 64 | 65 | // Computes count per path over a window 66 | .stage(new AggregationStage(), AggregationStage.config()) 67 | 68 | // Collects the data and makes it availabe over SSE 69 | .stage(new CollectStage(), CollectStage.config()) 70 | 71 | // Reuse built in sink that eagerly subscribes and delivers data over SSE 72 | .sink(Sinks.eagerSubscribe( 73 | Sinks.sse((String data) -> data))) 74 | 75 | .metadata(new Metadata.Builder() 76 | .name("GroupByPath") 77 | .description("Connects to a random data generator source" 78 | + " and counts the number of requests for each uri within a window") 79 | .build()) 80 | .create(); 81 | 82 | } 83 | 84 | public static void main(String[] args) { 85 | // To run locally we use the LocalJobExecutor 86 | LocalJobExecutorNetworked.execute(new RequestAggregationJob().getJobInstance()); 87 | } 88 | } -------------------------------------------------------------------------------- /groupby-sample/src/main/java/com/netflix/mantis/samples/proto/AggregationReport.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.samples.proto; 18 | 19 | import java.util.Map; 20 | 21 | import lombok.Data; 22 | 23 | 24 | /** 25 | * A simple POJO which holds the result of aggregating counts per request path. 26 | */ 27 | @Data 28 | public class AggregationReport { 29 | private final Map pathToCountMap; 30 | } 31 | -------------------------------------------------------------------------------- /groupby-sample/src/main/java/com/netflix/mantis/samples/proto/RequestAggregation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.samples.proto; 18 | 19 | import java.io.IOException; 20 | 21 | import com.fasterxml.jackson.databind.ObjectMapper; 22 | import com.fasterxml.jackson.databind.ObjectReader; 23 | import io.mantisrx.common.codec.Codec; 24 | import lombok.Builder; 25 | import lombok.Data; 26 | 27 | 28 | /** 29 | * A simple POJO that holds the count of how many times a particular request path was invoked. 30 | */ 31 | @Data 32 | @Builder 33 | public class RequestAggregation { 34 | private static final ObjectMapper mapper = new ObjectMapper(); 35 | private static final ObjectReader requestAggregationReader = mapper.readerFor(RequestAggregation.class); 36 | 37 | private final String path; 38 | private final int count; 39 | 40 | 41 | /** 42 | * Codec is used to customize how data is serialized before transporting across network boundaries. 43 | * @return 44 | */ 45 | public static Codec requestAggregationCodec() { 46 | 47 | return new Codec() { 48 | @Override 49 | public RequestAggregation decode(byte[] bytes) { 50 | 51 | try { 52 | return requestAggregationReader.readValue(bytes); 53 | } catch (IOException e) { 54 | throw new RuntimeException(e); 55 | } 56 | } 57 | 58 | @Override 59 | public byte[] encode(final RequestAggregation value) { 60 | 61 | try { 62 | return mapper.writeValueAsBytes(value); 63 | } catch (Exception e) { 64 | throw new RuntimeException(e); 65 | } 66 | } 67 | }; 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /groupby-sample/src/main/java/com/netflix/mantis/samples/proto/RequestEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.mantis.samples.proto; 17 | 18 | import java.io.IOException; 19 | 20 | import io.mantisrx.common.codec.Codec; 21 | import com.fasterxml.jackson.databind.ObjectMapper; 22 | import com.fasterxml.jackson.databind.ObjectReader; 23 | import lombok.Builder; 24 | import lombok.Data; 25 | 26 | 27 | /** 28 | * A simple POJO that holds data about a request event. 29 | */ 30 | @Data 31 | @Builder 32 | public class RequestEvent { 33 | private static final ObjectMapper mapper = new ObjectMapper(); 34 | private static final ObjectReader requestEventReader = mapper.readerFor(RequestEvent.class); 35 | 36 | private final String requestPath; 37 | private final String ipAddress; 38 | 39 | /** 40 | * The codec defines how this class should be serialized before transporting across network. 41 | * @return 42 | */ 43 | public static Codec requestEventCodec() { 44 | 45 | return new Codec() { 46 | @Override 47 | public RequestEvent decode(byte[] bytes) { 48 | 49 | try { 50 | return requestEventReader.readValue(bytes); 51 | } catch (IOException e) { 52 | throw new RuntimeException(e); 53 | } 54 | } 55 | 56 | @Override 57 | public byte[] encode(final RequestEvent value) { 58 | 59 | try { 60 | return mapper.writeValueAsBytes(value); 61 | } catch (Exception e) { 62 | throw new RuntimeException(e); 63 | } 64 | } 65 | }; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /groupby-sample/src/main/java/com/netflix/mantis/samples/source/RandomRequestSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.samples.source; 18 | 19 | import java.util.concurrent.TimeUnit; 20 | 21 | import com.netflix.mantis.samples.proto.RequestEvent; 22 | import io.mantisrx.runtime.Context; 23 | import io.mantisrx.runtime.source.Index; 24 | import io.mantisrx.runtime.source.Source; 25 | import lombok.extern.slf4j.Slf4j; 26 | import net.andreinc.mockneat.MockNeat; 27 | import rx.Observable; 28 | 29 | 30 | /** 31 | * Generates random set of RequestEvents at a preconfigured interval. 32 | */ 33 | @Slf4j 34 | public class RandomRequestSource implements Source { 35 | private MockNeat mockDataGenerator; 36 | 37 | @Override 38 | public Observable> call(Context context, Index index) { 39 | return Observable.just(Observable.interval(250, TimeUnit.MILLISECONDS).map((tick) -> { 40 | String ip = mockDataGenerator.ipv4s().get(); 41 | String path = mockDataGenerator.probabilites(String.class) 42 | .add(0.1, "/login") 43 | .add(0.2, "/genre/horror") 44 | .add(0.5, "/genre/comedy") 45 | .add(0.2, "/mylist") 46 | .get(); 47 | return RequestEvent.builder().ipAddress(ip).requestPath(path).build(); 48 | }).doOnNext((event) -> { 49 | log.debug("Generated Event {}", event); 50 | })); 51 | 52 | } 53 | 54 | @Override 55 | public void init(Context context, Index index) { 56 | mockDataGenerator = MockNeat.threadLocal(); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /groupby-sample/src/main/java/com/netflix/mantis/samples/stage/AggregationStage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.samples.stage; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | import java.util.concurrent.TimeUnit; 22 | 23 | import com.netflix.mantis.samples.proto.RequestAggregation; 24 | import com.netflix.mantis.samples.proto.RequestEvent; 25 | import io.mantisrx.common.MantisGroup; 26 | import io.mantisrx.common.codec.Codec; 27 | import io.mantisrx.runtime.Context; 28 | import io.mantisrx.runtime.GroupToScalar; 29 | import io.mantisrx.runtime.computation.GroupToScalarComputation; 30 | import io.mantisrx.runtime.parameter.ParameterDefinition; 31 | import io.mantisrx.runtime.parameter.type.IntParameter; 32 | import io.mantisrx.runtime.parameter.validator.Validators; 33 | import lombok.extern.slf4j.Slf4j; 34 | import rx.Observable; 35 | 36 | 37 | /** 38 | * This is the 2nd stage of this three stage job. It receives events from {@link GroupByStage} 39 | * This stage converts Grouped Events to Scalar events {@link GroupToScalarComputation} Typically used in the 40 | * reduce portion of a map reduce computation. 41 | * 42 | * This stage receives an Observable>. This represents a stream of 43 | * request events tagged by the URI Path they belong to. 44 | * This stage then groups the events by the path and and counts the number of invocations of each path over a window. 45 | */ 46 | @Slf4j 47 | public class AggregationStage implements GroupToScalarComputation { 48 | 49 | public static final String AGGREGATION_DURATION_MSEC_PARAM = "AggregationDurationMsec"; 50 | int aggregationDurationMsec; 51 | 52 | /** 53 | * The call method is invoked by the Mantis runtime while executing the job. 54 | * @param context Provides metadata information related to the current job. 55 | * @param mantisGroupO This is an Observable of {@link MantisGroup} events. Each event is a pair of the Key -> uri Path and 56 | * the {@link RequestEvent} event itself. 57 | * @return 58 | */ 59 | @Override 60 | public Observable call(Context context, Observable> mantisGroupO) { 61 | return mantisGroupO 62 | .window(aggregationDurationMsec, TimeUnit.MILLISECONDS) 63 | .flatMap((omg) -> omg.groupBy(MantisGroup::getKeyValue) 64 | .flatMap((go) -> go.reduce(0, (accumulator, value) -> accumulator = accumulator + 1) 65 | .map((count) -> RequestAggregation.builder().count(count).path(go.getKey()).build()) 66 | .doOnNext((aggregate) -> { 67 | log.debug("Generated aggregate {}", aggregate); 68 | }) 69 | )); 70 | } 71 | 72 | /** 73 | * Invoked only once during job startup. A good place to add one time initialization actions. 74 | * @param context 75 | */ 76 | @Override 77 | public void init(Context context) { 78 | aggregationDurationMsec = (int)context.getParameters().get(AGGREGATION_DURATION_MSEC_PARAM, 1000); 79 | } 80 | 81 | /** 82 | * Provides the Mantis runtime configuration information about the type of computation done by this stage. 83 | * E.g in this case it specifies this is a GroupToScalar computation and also provides a {@link Codec} on how to 84 | * serialize the {@link RequestAggregation} events before sending it to the {@link CollectStage} 85 | * @return 86 | */ 87 | public static GroupToScalar.Config config(){ 88 | return new GroupToScalar.Config() 89 | .description("sum events for a path") 90 | .codec(RequestAggregation.requestAggregationCodec()) 91 | .withParameters(getParameters()); 92 | } 93 | 94 | /** 95 | * Here we declare stage specific parameters. 96 | * @return 97 | */ 98 | public static List> getParameters() { 99 | List> params = new ArrayList<>(); 100 | 101 | // Aggregation duration 102 | params.add(new IntParameter() 103 | .name(AGGREGATION_DURATION_MSEC_PARAM) 104 | .description("window size for aggregation") 105 | .validator(Validators.range(100, 10000)) 106 | .defaultValue(5000) 107 | .build()) ; 108 | 109 | return params; 110 | } 111 | 112 | } 113 | -------------------------------------------------------------------------------- /groupby-sample/src/main/java/com/netflix/mantis/samples/stage/CollectStage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.samples.stage; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | import java.util.Objects; 22 | import java.util.concurrent.TimeUnit; 23 | 24 | import com.fasterxml.jackson.core.JsonProcessingException; 25 | import com.fasterxml.jackson.databind.ObjectMapper; 26 | import com.netflix.mantis.samples.proto.RequestAggregation; 27 | import com.netflix.mantis.samples.proto.AggregationReport; 28 | import io.mantisrx.common.codec.Codecs; 29 | import io.mantisrx.runtime.Context; 30 | import io.mantisrx.runtime.ScalarToScalar; 31 | import io.mantisrx.runtime.computation.ScalarComputation; 32 | import lombok.extern.slf4j.Slf4j; 33 | import rx.Observable; 34 | 35 | 36 | /** 37 | * This is the final stage of this 3 stage job. It receives events from {@link AggregationStage} 38 | * The role of this stage is to collect aggregates generated by the previous stage for all the groups within 39 | * a window and generate a unified report of them. 40 | */ 41 | @Slf4j 42 | public class CollectStage implements ScalarComputation { 43 | private static final ObjectMapper mapper = new ObjectMapper(); 44 | @Override 45 | public Observable call(Context context, Observable requestAggregationO) { 46 | return requestAggregationO 47 | .window(5, TimeUnit.SECONDS) 48 | .flatMap((requestAggO) -> requestAggO 49 | .reduce(new RequestAggregationAccumulator(),(acc, requestAgg) -> acc.addAggregation(requestAgg)) 50 | .map(RequestAggregationAccumulator::generateReport) 51 | .doOnNext((report) -> { 52 | log.debug("Generated Collection report {}", report); 53 | }) 54 | ) 55 | .map((report) -> { 56 | try { 57 | return mapper.writeValueAsString(report); 58 | } catch (JsonProcessingException e) { 59 | log.error(e.getMessage()); 60 | return null; 61 | } 62 | }).filter(Objects::nonNull); 63 | } 64 | 65 | @Override 66 | public void init(Context context) { 67 | 68 | } 69 | 70 | public static ScalarToScalar.Config config(){ 71 | return new ScalarToScalar.Config() 72 | .codec(Codecs.string()); 73 | } 74 | 75 | /** 76 | * The accumulator class as the name suggests accumulates all aggregates seen during a window and 77 | * generates a consolidated report at the end. 78 | */ 79 | static class RequestAggregationAccumulator { 80 | private final Map pathToCountMap = new HashMap<>(); 81 | 82 | public RequestAggregationAccumulator() {} 83 | 84 | public RequestAggregationAccumulator addAggregation(RequestAggregation agg) { 85 | pathToCountMap.put(agg.getPath(), agg.getCount()); 86 | return this; 87 | } 88 | 89 | public AggregationReport generateReport() { 90 | log.info("Generated report from=> {}", pathToCountMap); 91 | return new AggregationReport(pathToCountMap); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /groupby-sample/src/main/java/com/netflix/mantis/samples/stage/GroupByStage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.samples.stage; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | import com.netflix.mantis.samples.proto.RequestEvent; 23 | import io.mantisrx.common.MantisGroup; 24 | import io.mantisrx.runtime.Context; 25 | import io.mantisrx.runtime.ScalarToGroup; 26 | import io.mantisrx.runtime.computation.ToGroupComputation; 27 | import io.mantisrx.runtime.parameter.ParameterDefinition; 28 | import io.mantisrx.runtime.parameter.type.StringParameter; 29 | import io.mantisrx.runtime.parameter.validator.Validators; 30 | import lombok.extern.slf4j.Slf4j; 31 | import rx.Observable; 32 | 33 | 34 | /** 35 | * This is the first stage of this 3 stage job. It is at the head of the computation DAG 36 | * This stage converts Scalar Events to Grouped Events {@link ToGroupComputation}. The grouped events are then 37 | * send to the next stage of the Mantis Job in a way such that all events belonging to a particular group will 38 | * land on the same worker of the next stage. 39 | * 40 | * It receives a stream of {@link RequestEvent} and groups them by either the path or the IP address 41 | * based on the parameters passed by the user. 42 | */ 43 | @Slf4j 44 | public class GroupByStage implements ToGroupComputation { 45 | 46 | private static final String GROUPBY_FIELD_PARAM = "groupByField"; 47 | private boolean groupByPath = true; 48 | @Override 49 | public Observable> call(Context context, Observable requestEventO) { 50 | return requestEventO 51 | .map((requestEvent) -> { 52 | if(groupByPath) { 53 | return new MantisGroup<>(requestEvent.getRequestPath(), requestEvent); 54 | } else { 55 | return new MantisGroup<>(requestEvent.getIpAddress(), requestEvent); 56 | } 57 | }); 58 | } 59 | 60 | @Override 61 | public void init(Context context) { 62 | String groupByField = (String)context.getParameters().get(GROUPBY_FIELD_PARAM,"path"); 63 | groupByPath = groupByField.equalsIgnoreCase("path") ? true : false; 64 | } 65 | 66 | /** 67 | * Here we declare stage specific parameters. 68 | * @return 69 | */ 70 | public static List> getParameters() { 71 | List> params = new ArrayList<>(); 72 | 73 | // Group by field 74 | params.add(new StringParameter() 75 | .name(GROUPBY_FIELD_PARAM) 76 | .description("The key to group events by") 77 | .validator(Validators.notNullOrEmpty()) 78 | .defaultValue("path") 79 | .build()) ; 80 | 81 | return params; 82 | } 83 | 84 | public static ScalarToGroup.Config config(){ 85 | return new ScalarToGroup.Config() 86 | .description("Group event data by path/ip") 87 | .concurrentInput() // signifies events can be processed parallely 88 | .withParameters(getParameters()) 89 | .codec(RequestEvent.requestEventCodec()); 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /groupby-sample/src/main/resources/META-INF/services/io.mantisrx.runtime.MantisJobProvider: -------------------------------------------------------------------------------- 1 | com.netflix.mantis.samples.RequestAggregationJob -------------------------------------------------------------------------------- /groupby-sample/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /installViaTravis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script will install the project. 3 | 4 | if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then 5 | echo -e "Assemble Pull Request #$TRAVIS_PULL_REQUEST => Branch [$TRAVIS_BRANCH]" 6 | ./gradlew assemble --info 7 | elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" == "" ]; then 8 | echo -e 'Assemble Branch with Snapshot => Branch ['$TRAVIS_BRANCH']' 9 | ./gradlew -Prelease.travisci=true assemble --info 10 | elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" != "" ]; then 11 | echo -e 'Assemble Branch for Release => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG']' 12 | ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true assemble --info 13 | else 14 | echo -e 'WARN: Should not be here => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG'] Pull Request ['$TRAVIS_PULL_REQUEST']' 15 | ./gradlew assemble --info 16 | fi 17 | -------------------------------------------------------------------------------- /jobconnector-sample/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | 3 | configurations.all { 4 | resolutionStrategy { 5 | force "com.google.guava:guava:18.0" 6 | 7 | } 8 | } 9 | 10 | task execute(type:JavaExec) { 11 | main = "com.netflix.mantis.samples.JobConnectorJob" 12 | classpath = sourceSets.main.runtimeClasspath 13 | } 14 | 15 | ext { 16 | mantisVersion = '1.2.+' 17 | mantisConnectorsVersion = '1.2.5' 18 | slf4jVersion = '1.7.28' 19 | 20 | } 21 | 22 | dependencies { 23 | implementation "io.mantisrx:mantis-runtime:$mantisVersion" 24 | implementation "io.mantisrx:mantis-connector-job:$mantisConnectorsVersion" 25 | implementation "org.slf4j:slf4j-log4j12:$slf4jVersion" 26 | compileOnly "org.projectlombok:lombok:1.16.16" 27 | 28 | } 29 | 30 | 31 | -------------------------------------------------------------------------------- /jobconnector-sample/src/main/java/com/netflix/mantis/samples/JobConnectorJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.samples; 18 | 19 | import java.util.ArrayList; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | import com.fasterxml.jackson.core.JsonProcessingException; 25 | import com.fasterxml.jackson.databind.ObjectMapper; 26 | import com.netflix.mantis.samples.stage.EchoStage; 27 | 28 | import io.mantisrx.connector.job.core.MantisSourceJobConnector; 29 | import io.mantisrx.connector.job.source.JobSource; 30 | import io.mantisrx.runtime.Job; 31 | import io.mantisrx.runtime.MantisJob; 32 | import io.mantisrx.runtime.MantisJobProvider; 33 | import io.mantisrx.runtime.Metadata; 34 | import io.mantisrx.runtime.executor.LocalJobExecutorNetworked; 35 | import io.mantisrx.runtime.parameter.Parameter; 36 | import io.mantisrx.runtime.sink.Sinks; 37 | import lombok.extern.slf4j.Slf4j; 38 | 39 | 40 | /** 41 | * This sample demonstrates how to connect to the output of another job using the {@link JobSource} 42 | * If the target job is a source job then you can request a filtered stream of events from the source job 43 | * by passing an MQL query. 44 | * In this example we connect to the latest running instance of SyntheticSourceJob using the query 45 | * select country from stream where status==500 and simply echo the output. 46 | * 47 | * Run this sample by executing the main method of this class. Then look for the SSE port where the output of this job 48 | * will be available for streaming. E.g Serving modern HTTP SSE server sink on port: 8299 49 | * via command line do ../gradlew execute 50 | * 51 | * Note: this sample may not work in your IDE as the Mantis runtime needs to discover the location of the 52 | * SyntheticSourceJob. 53 | */ 54 | 55 | @Slf4j 56 | public class JobConnectorJob extends MantisJobProvider { 57 | 58 | @Override 59 | public Job getJobInstance() { 60 | 61 | return MantisJob 62 | // Stream Events from a job specified via job parameters 63 | .source(new JobSource()) 64 | 65 | // Simple echoes the data 66 | .stage(new EchoStage(), EchoStage.config()) 67 | 68 | // Reuse built in sink that eagerly subscribes and delivers data over SSE 69 | .sink(Sinks.eagerSubscribe( 70 | Sinks.sse((String data) -> data))) 71 | 72 | .metadata(new Metadata.Builder() 73 | .name("ConnectToJob") 74 | .description("Connects to the output of another job" 75 | + " and simply echoes the data") 76 | .build()) 77 | .create(); 78 | 79 | } 80 | 81 | public static void main(String[] args) throws JsonProcessingException { 82 | Map targetMap = new HashMap<>(); 83 | List targetInfos = new ArrayList<>(); 84 | 85 | JobSource.TargetInfo targetInfo = new JobSource.TargetInfoBuilder().withClientId("abc") 86 | .withSourceJobName("SyntheticSourceJob") 87 | .withQuery("select country from stream where status==500") 88 | .build(); 89 | targetInfos.add(targetInfo); 90 | targetMap.put("targets",targetInfos); 91 | ObjectMapper mapper = new ObjectMapper(); 92 | String target = mapper.writeValueAsString(targetMap); 93 | 94 | // To run locally we use the LocalJobExecutor 95 | LocalJobExecutorNetworked.execute(new JobConnectorJob().getJobInstance(), 96 | new Parameter(MantisSourceJobConnector.MANTIS_SOURCEJOB_TARGET_KEY, target)); 97 | } 98 | } -------------------------------------------------------------------------------- /jobconnector-sample/src/main/java/com/netflix/mantis/samples/stage/EchoStage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.samples.stage; 18 | 19 | import com.fasterxml.jackson.databind.ObjectMapper; 20 | import io.mantisrx.common.MantisServerSentEvent; 21 | import io.mantisrx.common.codec.Codecs; 22 | import io.mantisrx.runtime.Context; 23 | import io.mantisrx.runtime.ScalarToScalar; 24 | import io.mantisrx.runtime.computation.ScalarComputation; 25 | import lombok.extern.slf4j.Slf4j; 26 | import rx.Observable; 27 | 28 | 29 | /** 30 | * A simple stage that extracts data from the incoming {@link MantisServerSentEvent} and echoes it. 31 | */ 32 | @Slf4j 33 | public class EchoStage implements ScalarComputation { 34 | private static final ObjectMapper mapper = new ObjectMapper(); 35 | @Override 36 | public Observable call(Context context, Observable eventsO) { 37 | return eventsO 38 | .map(MantisServerSentEvent::getEventAsString) 39 | .map((event) -> { 40 | log.info("Received: {}", event); 41 | return event; 42 | }); 43 | } 44 | 45 | @Override 46 | public void init(Context context) { 47 | 48 | } 49 | 50 | public static ScalarToScalar.Config config(){ 51 | return new ScalarToScalar.Config() 52 | .codec(Codecs.string()); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /jobconnector-sample/src/main/resources/META-INF/services/io.mantisrx.runtime.MantisJobProvider: -------------------------------------------------------------------------------- 1 | com.netflix.mantis.samples.JobConnectorJob -------------------------------------------------------------------------------- /jobconnector-sample/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /mantis-publish-sample/Dockerfile: -------------------------------------------------------------------------------- 1 | # Alpine Linux with OpenJDK JRE 2 | FROM openjdk:8-jre-alpine 3 | # copy WAR into image 4 | 5 | RUN mkdir -p /mnt/local/mantispublish 6 | #RUN mkdir -p /mnt/local/mantispublish/mantis-examples-mantis-publish-sample-0.1.0-SNAPSHOT 7 | #RUN mkdir -p /mnt/local/mantispublish/mantis-examples-mantis-publish-sample-0.1.0-SNAPSHOT/lib 8 | #RUN mkdir -p /mnt/local/mantispublish/mantis-examples-mantis-publish-sample-0.1.0-SNAPSHOT/bin 9 | 10 | # NOTE: Assumes you're building in the mantis-server-worker directory 11 | COPY ./build/distributions/mantis-examples-mantis-publish-sample-shadow-0.1.0-SNAPSHOT/lib/mantis-examples-mantis-publish-sample-0.1.0-SNAPSHOT-all.jar /mnt/local/mantispublish/ 12 | #COPY ./build/distributions/mantis-publish-netty-guice-shadow-1.3.0-SNAPSHOT.zip /mnt/local/mantispublish/ 13 | #RUN unzip /mnt/local/mantispublish/mantis-examples-mantis-publish-sample-0.1.0-SNAPSHOT.zip 14 | 15 | WORKDIR /mnt/local/mantispublish/ 16 | # run application with this command line 17 | #CMD ["./mantis-examples-mantis-publish-sample"] 18 | CMD ["/usr/bin/java", "-jar", "/mnt/local/mantispublish/mantis-examples-mantis-publish-sample-0.1.0-SNAPSHOT-all.jar"] 19 | #CMD ls -lah /mnt/local/mantispublish/mantis-examples-mantis-publish-sample-0.1.0-SNAPSHOT/bin 20 | -------------------------------------------------------------------------------- /mantis-publish-sample/README.md: -------------------------------------------------------------------------------- 1 | #Mantis Publish Sample 2 | 3 | ## Introduction 4 | 5 | ## Requirements 6 | 7 | ## How to run 8 | 9 | ### Updating docker compose 10 | 11 | ```bash 12 | mantispublish: 13 | image: dev/mantispublish 14 | ports: 15 | - "8101:8101" 16 | depends_on: 17 | - mantisapi 18 | networks: 19 | mantis_net: 20 | ipv4_address: 172.16.186.8 21 | 22 | ``` -------------------------------------------------------------------------------- /mantis-publish-sample/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | apply plugin: 'java' 18 | apply plugin: 'application' 19 | apply plugin: 'com.github.johnrengelman.shadow' 20 | repositories { 21 | maven { 22 | url "https://dl.bintray.com/netflixoss/maven" 23 | } 24 | } 25 | 26 | if (project.hasProperty('useMavenLocal')) { 27 | repositories { 28 | mavenLocal() 29 | } 30 | } 31 | 32 | buildscript { 33 | repositories { 34 | jcenter() 35 | } 36 | dependencies { 37 | classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' 38 | } 39 | } 40 | application { 41 | mainClassName = 'com.netflix.mantis.examples.mantispublishsample.Application' 42 | } 43 | 44 | ext { 45 | 46 | guiceVersion = '4.2.2' 47 | spectatorVersion = '0.96.0' 48 | archaiusVersion = 'latest.release' 49 | mockNeatVersion = '0.3.7' 50 | slf4jVersion = '1.7.28' 51 | lombokVersion = '1.16.16' 52 | } 53 | 54 | 55 | dependencies { 56 | compile 'io.mantisrx:mantis-publish-netty-guice:1.2.+' 57 | implementation "com.netflix.spectator:spectator-ext-ipc:$spectatorVersion" 58 | implementation "com.netflix.archaius:archaius2-core:$archaiusVersion" 59 | implementation "com.netflix.archaius:archaius2-guice:$archaiusVersion" 60 | implementation "com.google.inject:guice:$guiceVersion" 61 | implementation "com.netflix.spectator:spectator-nflx-plugin:$spectatorVersion" 62 | implementation "net.andreinc.mockneat:mockneat:$mockNeatVersion" 63 | 64 | implementation "org.slf4j:slf4j-log4j12:$slf4jVersion" 65 | compileOnly "org.projectlombok:lombok:$lombokVersion" 66 | } 67 | 68 | -------------------------------------------------------------------------------- /mantis-publish-sample/buildDockerImage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # build the mantis-server-worker fat jar 4 | ../gradlew clean build shadowDistZip 5 | unzip build/distributions/mantis-examples-mantis-publish-sample-shadow-0.1.0-SNAPSHOT.zip -d build/distributions 6 | # build the Docker image that packages the mantis-server-worker along with a running mesos-slave 7 | docker build -t dev/mantispublish . 8 | 9 | echo "Created Docker image 'dev/mantispublish'" 10 | -------------------------------------------------------------------------------- /mantis-publish-sample/src/main/java/com/netflix/mantis/examples/mantispublishsample/Application.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.mantispublishsample; 18 | 19 | import com.google.inject.Guice; 20 | import com.google.inject.Injector; 21 | import com.netflix.archaius.guice.ArchaiusModule; 22 | import com.netflix.spectator.nflx.SpectatorModule; 23 | import io.mantisrx.publish.api.EventPublisher; 24 | import io.mantisrx.publish.netty.guice.MantisRealtimeEventsPublishModule; 25 | 26 | 27 | /** 28 | * A simple example that uses Guice to inject the {@link EventPublisher} part of the mantis-publish library 29 | * to send events to Mantis. 30 | * 31 | * The mantis-publish library provides on-demand source side filtering via MQL. When a user publishes 32 | * events via this library the events may not be actually shipped to Mantis. A downstream consumer needs 33 | * to first register a query and the query needs to match events published by the user. 34 | */ 35 | public class Application { 36 | 37 | public static void main(String [] args) { 38 | Injector injector = Guice.createInjector(new BasicModule(), new ArchaiusModule(), 39 | new MantisRealtimeEventsPublishModule(), new SpectatorModule()); 40 | 41 | IDataPublisher publisher = injector.getInstance(IDataPublisher.class); 42 | 43 | publisher.generateAndSendEventsToMantis(); 44 | 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /mantis-publish-sample/src/main/java/com/netflix/mantis/examples/mantispublishsample/BasicModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.mantispublishsample; 18 | 19 | import com.google.inject.AbstractModule; 20 | 21 | 22 | public class BasicModule extends AbstractModule { 23 | 24 | @Override 25 | protected void configure() { 26 | bind(IDataPublisher.class).to(SampleDataPublisher.class); 27 | bind(IDataGenerator.class).to(DataGenerator.class); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /mantis-publish-sample/src/main/java/com/netflix/mantis/examples/mantispublishsample/DataGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.mantispublishsample; 18 | 19 | import java.util.concurrent.TimeUnit; 20 | 21 | import com.netflix.mantis.examples.mantispublishsample.proto.RequestEvent; 22 | import net.andreinc.mockneat.MockNeat; 23 | import rx.Observable; 24 | 25 | 26 | /** 27 | * Uses MockNeat to generate a random stream of events. Each event represents a hypothetical request 28 | * made by an end user to this service. 29 | */ 30 | public class DataGenerator implements IDataGenerator { 31 | private final int rateMs = 1000; 32 | private final MockNeat mockDataGenerator = MockNeat.threadLocal(); 33 | 34 | @Override 35 | public Observable generateEvents() { 36 | return Observable 37 | .interval(rateMs, TimeUnit.MILLISECONDS) 38 | .map((tick) -> generateEvent()); 39 | } 40 | 41 | private RequestEvent generateEvent() { 42 | 43 | String path = mockDataGenerator.probabilites(String.class) 44 | .add(0.1, "/login") 45 | .add(0.2, "/genre/horror") 46 | .add(0.5, "/genre/comedy") 47 | .add(0.2, "/mylist") 48 | .get(); 49 | 50 | String deviceType = mockDataGenerator.probabilites(String.class) 51 | .add(0.1, "ps4") 52 | .add(0.1, "xbox") 53 | .add(0.2, "browser") 54 | .add(0.3, "ios") 55 | .add(0.3, "android") 56 | .get(); 57 | String userId = mockDataGenerator.strings().size(10).get(); 58 | int status = mockDataGenerator.probabilites(Integer.class) 59 | .add(0.1,500) 60 | .add(0.7,200) 61 | .add(0.2,500) 62 | .get(); 63 | 64 | 65 | String country = mockDataGenerator.countries().names().get(); 66 | return RequestEvent.builder() 67 | .status(status) 68 | .uri(path) 69 | .country(country) 70 | .userId(userId) 71 | .deviceType(deviceType) 72 | .build(); 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /mantis-publish-sample/src/main/java/com/netflix/mantis/examples/mantispublishsample/IDataGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.mantispublishsample; 18 | 19 | import com.netflix.mantis.examples.mantispublishsample.proto.RequestEvent; 20 | import rx.Observable; 21 | 22 | 23 | /** 24 | * A data generator that generates a stream of {@link RequestEvent} at a fixed interval. 25 | */ 26 | public interface IDataGenerator { 27 | 28 | Observable generateEvents(); 29 | } 30 | -------------------------------------------------------------------------------- /mantis-publish-sample/src/main/java/com/netflix/mantis/examples/mantispublishsample/IDataPublisher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.mantispublishsample; 18 | 19 | public interface IDataPublisher { 20 | void generateAndSendEventsToMantis(); 21 | } 22 | -------------------------------------------------------------------------------- /mantis-publish-sample/src/main/java/com/netflix/mantis/examples/mantispublishsample/SampleDataPublisher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.mantispublishsample; 18 | 19 | import com.google.inject.Inject; 20 | import io.mantisrx.publish.api.Event; 21 | import io.mantisrx.publish.api.EventPublisher; 22 | import io.mantisrx.publish.api.PublishStatus; 23 | import lombok.extern.slf4j.Slf4j; 24 | import rx.Observable; 25 | 26 | 27 | /** 28 | * A simple example that uses Guice to inject the {@link EventPublisher} part of the mantis-publish library 29 | * to send events to Mantis. 30 | * 31 | * The mantis-publish library provides on-demand source side filtering via MQL. When a user publishes 32 | * events via this library the events may not be actually shipped to Mantis. A downstream consumer needs 33 | * to first register a query and the query needs to match events published by the user. 34 | * 35 | */ 36 | @Slf4j 37 | public class SampleDataPublisher implements IDataPublisher{ 38 | @Inject 39 | EventPublisher publisher; 40 | @Inject 41 | DataGenerator dataGenerator; 42 | 43 | /** 44 | * Generates random events at a fixed rate and publishes them to the mantis-publish library. 45 | * Here the events are published to the defaultStream. 46 | */ 47 | @Override 48 | public void generateAndSendEventsToMantis() { 49 | dataGenerator 50 | .generateEvents() 51 | .map((requestEvent) -> new Event(requestEvent.toMap())) 52 | .flatMap((event) -> Observable.from(publisher.publish(event) 53 | .toCompletableFuture())) 54 | .toBlocking() 55 | .subscribe((status) -> { 56 | log.info("Mantis publish JavaApp send event status => {}", status); 57 | }); 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /mantis-publish-sample/src/main/java/com/netflix/mantis/examples/mantispublishsample/proto/RequestEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.mantispublishsample.proto; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import com.fasterxml.jackson.core.JsonProcessingException; 23 | import com.fasterxml.jackson.databind.ObjectMapper; 24 | import com.fasterxml.jackson.databind.ObjectReader; 25 | import lombok.Builder; 26 | import lombok.Data; 27 | 28 | 29 | /** 30 | * Represents a Request Event a service may receive. 31 | */ 32 | @Data 33 | @Builder 34 | public class RequestEvent { 35 | 36 | private static final ObjectMapper mapper = new ObjectMapper(); 37 | private static final ObjectReader requestEventReader = mapper.readerFor(RequestEvent.class); 38 | 39 | private final String userId; 40 | private final String uri; 41 | private final int status; 42 | private final String country; 43 | private final String deviceType; 44 | 45 | public Map toMap() { 46 | Map data = new HashMap<>(); 47 | data.put("userId", userId); 48 | data.put("uri", uri); 49 | data.put("status", status); 50 | data.put("country", country); 51 | data.put("deviceType", deviceType); 52 | return data; 53 | } 54 | 55 | public String toJsonString() { 56 | try { 57 | return mapper.writeValueAsString(this); 58 | } catch (JsonProcessingException e) { 59 | e.printStackTrace(); 60 | return null; 61 | } 62 | } 63 | 64 | 65 | 66 | } 67 | -------------------------------------------------------------------------------- /mantis-publish-sample/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | #For use in local docker setup 2 | #location of the mantis api server 3 | mantis.publish.discovery.api.hostname=172.16.186.7 4 | 5 | # mantis api port 6 | mantis.publish.discovery.api.port=7101 7 | 8 | # This application's name 9 | mantis.publish.app.name=JavaApp -------------------------------------------------------------------------------- /mantis-publish-sample/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /mantis-publish-web-sample/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tomcat:9.0-alpine 2 | 3 | LABEL maintainer="mantis-oss-dev@netflix.com" 4 | 5 | ADD build/libs/mantis-examples-mantis-publish-web-sample-0.1.0-SNAPSHOT.war /usr/local/tomcat/webapps/ 6 | 7 | EXPOSE 8080 8 | 9 | CMD ["catalina.sh", "run"] 10 | -------------------------------------------------------------------------------- /mantis-publish-web-sample/README.md: -------------------------------------------------------------------------------- 1 | #Web Mantis Publish Sample 2 | 3 | ## Introduction 4 | 5 | ## Requirements 6 | 7 | ## How to run 8 | 9 | ### Updating docker compose 10 | 11 | ```bash 12 | mantispublish: 13 | image: dev/mantispublishweb 14 | ports: 15 | - "80:8080" 16 | depends_on: 17 | - mantisapi 18 | networks: 19 | mantis_net: 20 | ipv4_address: 172.16.186.8 21 | 22 | ``` -------------------------------------------------------------------------------- /mantis-publish-web-sample/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | buildscript { 19 | repositories { 20 | jcenter() 21 | maven { 22 | url "https://dl.bintray.com/netflixoss/maven" 23 | } 24 | } 25 | } 26 | 27 | plugins { 28 | id 'war' 29 | id 'org.gretty' version '2.2.0' 30 | } 31 | 32 | 33 | ext { 34 | 35 | guiceVersion = '4.2.2' 36 | spectatorVersion = '0.96.0' 37 | archaiusVersion = 'latest.release' 38 | mockNeatVersion = '0.3.7' 39 | slf4jVersion = '1.7.28' 40 | lombokVersion = '1.16.16' 41 | guiceServletVersion = '4.0' 42 | } 43 | 44 | 45 | dependencies { 46 | 47 | providedCompile 'javax.servlet:javax.servlet-api:3.1.0' 48 | compile 'io.mantisrx:mantis-publish-netty-guice:1.2.+' 49 | implementation "com.netflix.spectator:spectator-ext-ipc:$spectatorVersion" 50 | implementation "com.netflix.archaius:archaius2-core:$archaiusVersion" 51 | implementation "com.netflix.archaius:archaius2-guice:$archaiusVersion" 52 | implementation "com.google.inject:guice:$guiceVersion" 53 | implementation "com.netflix.spectator:spectator-nflx-plugin:$spectatorVersion" 54 | implementation "org.slf4j:slf4j-log4j12:$slf4jVersion" 55 | 56 | compile 'com.google.inject.extensions:guice-servlet:4.0' 57 | compileOnly "org.projectlombok:lombok:$lombokVersion" 58 | } 59 | 60 | -------------------------------------------------------------------------------- /mantis-publish-web-sample/buildDockerImage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # build the mantis-server-worker fat jar 4 | ../gradlew clean build 5 | 6 | # build the Docker image that packages the mantis-server-worker along with a running mesos-slave 7 | docker build -t dev/mantispublishweb . 8 | 9 | echo "Created Docker image 'dev/mantispublishweb'" 10 | -------------------------------------------------------------------------------- /mantis-publish-web-sample/src/main/java/com/netflix/mantis/examples/mantispublishsample/web/config/DefaultGuiceServletConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.mantispublishsample.web.config; 18 | 19 | import com.google.inject.Guice; 20 | import com.google.inject.Injector; 21 | import com.google.inject.servlet.GuiceServletContextListener; 22 | import com.google.inject.servlet.ServletModule; 23 | import com.netflix.archaius.guice.ArchaiusModule; 24 | import com.netflix.mantis.examples.mantispublishsample.web.filter.CaptureRequestEventFilter; 25 | import com.netflix.mantis.examples.mantispublishsample.web.service.MyService; 26 | import com.netflix.mantis.examples.mantispublishsample.web.service.MyServiceImpl; 27 | import com.netflix.mantis.examples.mantispublishsample.web.servlet.HelloServlet; 28 | import com.netflix.spectator.nflx.SpectatorModule; 29 | import io.mantisrx.publish.netty.guice.MantisRealtimeEventsPublishModule; 30 | 31 | 32 | /** 33 | * Wire up the servlets, filters and other modules. 34 | */ 35 | public class DefaultGuiceServletConfig extends GuiceServletContextListener { 36 | @Override 37 | protected Injector getInjector() { 38 | return Guice.createInjector( 39 | new ArchaiusModule(), new MantisRealtimeEventsPublishModule(), new SpectatorModule(), 40 | new ServletModule() { 41 | @Override 42 | protected void configureServlets() { 43 | filter("/*").through(CaptureRequestEventFilter.class); 44 | serve("/hello").with(HelloServlet.class); 45 | bind(MyService.class).to(MyServiceImpl.class); 46 | } 47 | } 48 | ); 49 | } 50 | } -------------------------------------------------------------------------------- /mantis-publish-web-sample/src/main/java/com/netflix/mantis/examples/mantispublishsample/web/service/MyService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.mantispublishsample.web.service; 18 | 19 | public interface MyService { 20 | String hello(String name); 21 | } -------------------------------------------------------------------------------- /mantis-publish-web-sample/src/main/java/com/netflix/mantis/examples/mantispublishsample/web/service/MyServiceImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.mantispublishsample.web.service; 18 | 19 | public class MyServiceImpl implements MyService { 20 | @Override 21 | public String hello(String name) { 22 | return "Hello, " + name; 23 | } 24 | } -------------------------------------------------------------------------------- /mantis-publish-web-sample/src/main/java/com/netflix/mantis/examples/mantispublishsample/web/servlet/HelloServlet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.mantispublishsample.web.servlet; 18 | 19 | import java.io.IOException; 20 | 21 | import javax.inject.Inject; 22 | import javax.inject.Singleton; 23 | import javax.servlet.ServletException; 24 | import javax.servlet.http.HttpServlet; 25 | import javax.servlet.http.HttpServletRequest; 26 | import javax.servlet.http.HttpServletResponse; 27 | 28 | import com.netflix.mantis.examples.mantispublishsample.web.service.MyService; 29 | 30 | 31 | /** 32 | * A simple servlet that looks for the existence of a name parameter in the request and responds 33 | * with a Hello message. 34 | */ 35 | @Singleton 36 | public class HelloServlet extends HttpServlet { 37 | @Inject 38 | MyService myService; 39 | protected void doGet(HttpServletRequest request, HttpServletResponse response) 40 | throws ServletException, IOException { 41 | 42 | String name = request.getParameter("name"); 43 | if (name == null) name = "Universe"; 44 | String result = myService.hello(name); 45 | response.getWriter().print(result); 46 | } 47 | 48 | protected void doPost(HttpServletRequest request, HttpServletResponse response) 49 | throws ServletException, IOException { 50 | String name = request.getParameter("name"); 51 | if (name == null) name = "World"; 52 | request.setAttribute("user", name); 53 | request.getRequestDispatcher("response.jsp").forward(request, response); 54 | } 55 | } -------------------------------------------------------------------------------- /mantis-publish-web-sample/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | #For use in local docker setup 2 | #location of the mantis api server 3 | mantis.publish.discovery.api.hostname=172.16.186.7 4 | 5 | # mantis api port 6 | mantis.publish.discovery.api.port=7101 7 | 8 | # This application's name 9 | mantis.publish.app.name=TestApp -------------------------------------------------------------------------------- /mantis-publish-web-sample/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /mantis-publish-web-sample/src/main/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | 8 | 9 | com.netflix.mantis.examples.mantispublishsample.web.com.netflix.mantis.examples.config.DefaultGuiceServletConfig 10 | 11 | 12 | 13 | guiceFilter 14 | com.google.inject.servlet.GuiceFilter 15 | 16 | 17 | 18 | guiceFilter 19 | /* 20 | 21 | 22 | -------------------------------------------------------------------------------- /mantis-publish-web-sample/src/main/webapp/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Mantis Publish Demo 4 | 5 | 6 |

Hello ! What is your name?

7 | 8 |
9 |

Name:

10 | 11 | 12 |
13 | 14 | -------------------------------------------------------------------------------- /mantis-publish-web-sample/src/main/webapp/response.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html;charset=UTF-8" language="java" %> 2 | 3 | 4 | Hello Page 5 | 6 | 7 |

Hello, ${user}!

8 | 9 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | rootProject.name = 'mantis-examples' 18 | [ 19 | 'core', 20 | 'sine-function', 21 | 'wordcount', 22 | 'twitter-sample', 23 | 'groupby-sample', 24 | 'synthetic-sourcejob', 25 | 'jobconnector-sample', 26 | 'mantis-publish-sample', 27 | 'mantis-publish-web-sample' 28 | ].each { 29 | include "${it}" 30 | project(":${it}").name = "${rootProject.name}-${it}".replace('/', '-') 31 | } 32 | -------------------------------------------------------------------------------- /sine-function/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | configurations.all { 18 | resolutionStrategy { 19 | force "com.google.guava:guava:18.0" 20 | } 21 | } 22 | task execute(type:JavaExec) { 23 | 24 | main = "io.mantisrx.mantis.examples.sinefunction.SineFunctionJob" 25 | 26 | classpath = sourceSets.main.runtimeClasspath 27 | } 28 | dependencies { 29 | compile 'io.mantisrx:mantis-runtime:1.2.+' 30 | compile 'com.fasterxml.jackson.core:jackson-core:2.11.1' 31 | compile 'com.fasterxml.jackson.core:jackson-databind:2.11.1' 32 | } 33 | -------------------------------------------------------------------------------- /sine-function/src/main/java/io/mantisrx/mantis/examples/sinefunction/SineFunctionJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.mantis.examples.sinefunction; 18 | 19 | import java.util.Random; 20 | import java.util.concurrent.TimeUnit; 21 | 22 | import io.mantisrx.mantis.examples.sinefunction.core.Point; 23 | import io.mantisrx.mantis.examples.sinefunction.stages.SinePointGeneratorStage; 24 | import io.mantisrx.runtime.Context; 25 | import io.mantisrx.runtime.Job; 26 | import io.mantisrx.runtime.MantisJob; 27 | import io.mantisrx.runtime.MantisJobProvider; 28 | import io.mantisrx.runtime.Metadata; 29 | import io.mantisrx.runtime.ScalarToScalar; 30 | import io.mantisrx.runtime.codec.JacksonCodecs; 31 | import io.mantisrx.runtime.executor.LocalJobExecutorNetworked; 32 | import io.mantisrx.runtime.parameter.Parameter; 33 | import io.mantisrx.runtime.parameter.type.BooleanParameter; 34 | import io.mantisrx.runtime.parameter.type.DoubleParameter; 35 | import io.mantisrx.runtime.parameter.type.IntParameter; 36 | import io.mantisrx.runtime.parameter.validator.Validators; 37 | import io.mantisrx.runtime.sink.SelfDocumentingSink; 38 | import io.mantisrx.runtime.sink.ServerSentEventsSink; 39 | import io.mantisrx.runtime.sink.predicate.Predicate; 40 | import io.mantisrx.runtime.source.Index; 41 | import io.mantisrx.runtime.source.Source; 42 | import rx.Observable; 43 | import rx.functions.Func1; 44 | import rx.schedulers.Schedulers; 45 | 46 | 47 | public class SineFunctionJob extends MantisJobProvider { 48 | 49 | public static final String INTERVAL_SEC = "intervalSec"; 50 | public static final String RANGE_MAX = "max"; 51 | public static final String RANGE_MIN = "min"; 52 | public static final String AMPLITUDE = "amplitude"; 53 | public static final String FREQUENCY = "frequency"; 54 | public static final String PHASE = "phase"; 55 | public static final String RANDOM_RATE = "randomRate"; 56 | public static final String USE_RANDOM_FLAG = "useRandom"; 57 | 58 | /** 59 | * The SSE sink sets up an SSE server that can be connected to using SSE clients(curl etc.) to see 60 | * a real-time stream of (x, y) tuples on a sine curve. 61 | */ 62 | private final SelfDocumentingSink sseSink = new ServerSentEventsSink.Builder() 63 | .withEncoder(point -> String.format("{\"x\": %f, \"y\": %f}", point.getX(), point.getY())) 64 | .withPredicate(new Predicate<>( 65 | "filter=even, returns even x parameters; filter=odd, returns odd x parameters.", 66 | parameters -> { 67 | Func1 filter = point -> { 68 | return true; 69 | }; 70 | if (parameters != null && parameters.containsKey("filter")) { 71 | String filterBy = parameters.get("filter").get(0); 72 | // create filter function based on parameter value 73 | filter = point -> { 74 | // filter by evens or odds for x values 75 | if ("even".equalsIgnoreCase(filterBy)) { 76 | return (point.getX() % 2 == 0); 77 | } else if ("odd".equalsIgnoreCase(filterBy)) { 78 | return (point.getX() % 2 != 0); 79 | } 80 | return true; // if not even/odd 81 | }; 82 | } 83 | return filter; 84 | } 85 | )) 86 | .build(); 87 | 88 | /** 89 | * The Stage com.netflix.mantis.examples.config defines how the output of the stage is serialized onto the next stage or sink. 90 | */ 91 | static ScalarToScalar.Config stageConfig() { 92 | return new ScalarToScalar.Config() 93 | .codec(JacksonCodecs.pojo(Point.class)); 94 | } 95 | 96 | /** 97 | * Run this in the IDE and look for 98 | * {@code AbstractServer:95 main - Rx server started at port: } in the console output. 99 | * Connect to the port using {@code curl localhost:} 100 | * to see a stream of (x, y) coordinates on a sine curve. 101 | */ 102 | public static void main(String[] args) { 103 | LocalJobExecutorNetworked.execute(new SineFunctionJob().getJobInstance(), 104 | new Parameter("useRandom", "false")); 105 | } 106 | 107 | @Override 108 | public Job getJobInstance() { 109 | return MantisJob 110 | // Define the data source for this job. 111 | .source(new TimerSource()) 112 | // Add stages to transform the event stream received from the Source. 113 | .stage(new SinePointGeneratorStage(), stageConfig()) 114 | // Define a sink to output the transformed stream over SSE or an external system like Cassandra, etc. 115 | .sink(sseSink) 116 | // Add Job parameters that can be passed in by the user when submitting a job. 117 | .parameterDefinition(new BooleanParameter() 118 | .name(USE_RANDOM_FLAG) 119 | .required() 120 | .description("If true, produce a random sequence of integers. If false," 121 | + " produce a sequence of integers starting at 0 and increasing by 1.") 122 | .build()) 123 | .parameterDefinition(new DoubleParameter() 124 | .name(RANDOM_RATE) 125 | .defaultValue(1.0) 126 | .description("The chance a random integer is generated, for the given period") 127 | .validator(Validators.range(0, 1)) 128 | .build()) 129 | .parameterDefinition(new IntParameter() 130 | .name(INTERVAL_SEC) 131 | .defaultValue(1) 132 | .description("Period at which to generate a random integer value to send to sine function") 133 | .validator(Validators.range(1, 60)) 134 | .build()) 135 | .parameterDefinition(new IntParameter() 136 | .name(RANGE_MIN) 137 | .defaultValue(0) 138 | .description("Minimun of random integer value") 139 | .validator(Validators.range(0, 100)) 140 | .build()) 141 | .parameterDefinition(new IntParameter() 142 | .name(RANGE_MAX) 143 | .defaultValue(100) 144 | .description("Maximum of random integer value") 145 | .validator(Validators.range(1, 100)) 146 | .build()) 147 | .parameterDefinition(new DoubleParameter() 148 | .name(AMPLITUDE) 149 | .defaultValue(10.0) 150 | .description("Amplitude for sine function") 151 | .validator(Validators.range(1, 100)) 152 | .build()) 153 | .parameterDefinition(new DoubleParameter() 154 | .name(FREQUENCY) 155 | .defaultValue(1.0) 156 | .description("Frequency for sine function") 157 | .validator(Validators.range(1, 100)) 158 | .build()) 159 | .parameterDefinition(new DoubleParameter() 160 | .name(PHASE) 161 | .defaultValue(0.0) 162 | .description("Phase for sine function") 163 | .validator(Validators.range(0, 100)) 164 | .build()) 165 | .metadata(new Metadata.Builder() 166 | .name("Sine function") 167 | .description("Produces an infinite stream of points, along the sine function, using the" 168 | + " following function definition: f(x) = amplitude * sin(frequency * x + phase)." 169 | + " The input to the function is either random between [min, max], or an integer sequence starting " 170 | + " at 0. The output is served via HTTP server using SSE protocol.") 171 | .build()) 172 | .create(); 173 | } 174 | 175 | /** 176 | * This source generates a monotonically increasingly value per tick as per INTERVAL_SEC Job parameter. 177 | * If USE_RANDOM_FLAG is set, the source generates a random value per tick. 178 | */ 179 | class TimerSource implements Source { 180 | 181 | @Override 182 | public Observable> call(Context context, Index index) { 183 | // If you want to be informed of scaleup/scale down of the source stage of this job you can subscribe 184 | // to getTotalNumWorkersObservable like the following. 185 | index.getTotalNumWorkersObservable().subscribeOn(Schedulers.io()).subscribe((workerCount) -> { 186 | System.out.println("Total worker count changed to -> " + workerCount); 187 | }); 188 | final int period = (int) 189 | context.getParameters().get(INTERVAL_SEC); 190 | final int max = (int) 191 | context.getParameters().get(RANGE_MAX); 192 | final int min = (int) 193 | context.getParameters().get(RANGE_MIN); 194 | final double randomRate = (double) 195 | context.getParameters().get(RANDOM_RATE); 196 | final boolean useRandom = (boolean) 197 | context.getParameters().get(USE_RANDOM_FLAG); 198 | 199 | final Random randomNumGenerator = new Random(); 200 | final Random randomRateVariable = new Random(); 201 | 202 | return Observable.just( 203 | Observable.interval(0, period, TimeUnit.SECONDS) 204 | .map(time -> { 205 | if (useRandom) { 206 | return randomNumGenerator.nextInt((max - min) + 1) + min; 207 | } else { 208 | return (int) (long) time; 209 | } 210 | }) 211 | .filter(x -> { 212 | double value = randomRateVariable.nextDouble(); 213 | return (value <= randomRate); 214 | }) 215 | ); 216 | } 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /sine-function/src/main/java/io/mantisrx/mantis/examples/sinefunction/core/Point.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.mantis.examples.sinefunction.core; 18 | 19 | 20 | import com.fasterxml.jackson.annotation.JsonCreator; 21 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 22 | import com.fasterxml.jackson.annotation.JsonProperty; 23 | import io.mantisrx.runtime.codec.JsonType; 24 | 25 | 26 | public class Point implements JsonType { 27 | 28 | private double x; 29 | private double y; 30 | 31 | @JsonCreator 32 | @JsonIgnoreProperties(ignoreUnknown = true) 33 | public Point(@JsonProperty("x") double x, 34 | @JsonProperty("y") double y) { 35 | this.x = x; 36 | this.y = y; 37 | } 38 | 39 | public double getX() { 40 | return x; 41 | } 42 | 43 | public double getY() { 44 | return y; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /sine-function/src/main/java/io/mantisrx/mantis/examples/sinefunction/stages/SinePointGeneratorStage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.mantis.examples.sinefunction.stages; 18 | 19 | import io.mantisrx.mantis.examples.sinefunction.SineFunctionJob; 20 | import io.mantisrx.mantis.examples.sinefunction.core.Point; 21 | import io.mantisrx.runtime.Context; 22 | import io.mantisrx.runtime.computation.ScalarComputation; 23 | import rx.Observable; 24 | 25 | 26 | /** 27 | * This class implements the ScalarComputation type of Mantis Stage and 28 | * transforms the value received from the Source into a Point on a sine-function curve 29 | * based on AMPLITUDE, FREQUENCY and PHASE job parameters. 30 | */ 31 | public class SinePointGeneratorStage implements ScalarComputation { 32 | 33 | @Override 34 | public Observable call(Context context, Observable o) { 35 | final double amplitude = (double) 36 | context.getParameters().get(SineFunctionJob.AMPLITUDE); 37 | final double frequency = (double) 38 | context.getParameters().get(SineFunctionJob.FREQUENCY); 39 | final double phase = (double) 40 | context.getParameters().get(SineFunctionJob.PHASE); 41 | return 42 | o 43 | .filter(x -> x % 2 == 0) 44 | .map(x -> new Point(x, amplitude * Math.sin((frequency * x) + phase))); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /sine-function/src/main/resources/META-INF/services/io.mantisrx.runtime.MantisJobProvider: -------------------------------------------------------------------------------- 1 | io.mantisrx.mantis.examples.sinefunction.SineFunctionJob 2 | -------------------------------------------------------------------------------- /sine-function/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /synthetic-sourcejob/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | ext { 18 | mantisVersion = '1.2.+' 19 | mqlVersion = '3.2.2' 20 | mockNeatVersion = '0.3.7' 21 | slf4jVersion = '1.7.28' 22 | } 23 | 24 | task execute(type:JavaExec) { 25 | 26 | main = "io.mantisrx.sourcejob.synthetic.SyntheticSourceJob" 27 | 28 | classpath = sourceSets.main.runtimeClasspath 29 | } 30 | 31 | 32 | dependencies { 33 | implementation "io.mantisrx:mantis-runtime:$mantisVersion" 34 | implementation "io.mantisrx:mql-jvm:$mqlVersion" 35 | implementation 'com.fasterxml.jackson.core:jackson-core:2.11.1' 36 | implementation 'com.fasterxml.jackson.core:jackson-databind:2.11.1' 37 | 38 | implementation "net.andreinc.mockneat:mockneat:$mockNeatVersion" 39 | implementation "org.slf4j:slf4j-log4j12:$slf4jVersion" 40 | compileOnly "org.projectlombok:lombok:1.16.16" 41 | 42 | testCompile "org.mockito:mockito-all:1.9.5" 43 | 44 | } 45 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/java/io/mantisrx/sourcejob/synthetic/SyntheticSourceJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.sourcejob.synthetic; 18 | 19 | import io.mantisrx.runtime.Job; 20 | import io.mantisrx.runtime.MantisJob; 21 | import io.mantisrx.runtime.MantisJobProvider; 22 | import io.mantisrx.runtime.executor.LocalJobExecutorNetworked; 23 | import io.mantisrx.sourcejob.synthetic.core.TaggedData; 24 | import io.mantisrx.sourcejob.synthetic.sink.QueryRequestPostProcessor; 25 | import io.mantisrx.sourcejob.synthetic.sink.QueryRequestPreProcessor; 26 | import io.mantisrx.sourcejob.synthetic.sink.TaggedDataSourceSink; 27 | import io.mantisrx.sourcejob.synthetic.source.SyntheticSource; 28 | import io.mantisrx.sourcejob.synthetic.stage.TaggingStage; 29 | 30 | /** 31 | * A sample queryable source job that generates synthetic request events. 32 | * Clients connect to this job via the Sink port using an MQL expression. The job then sends only the data 33 | * that matches the query to the client. The client can be another Mantis Job or a user manually running a GET request. 34 | * 35 | * Run this sample by executing the main method of this class. Then look for the SSE port where the output of this job 36 | * will be available for streaming. E.g Serving modern HTTP SSE server sink on port: 8299 37 | * Usage: curl "localhost:?clientId=&subscriptionId=&criterion= 38 | * 39 | * E.g curl "localhost:8498?subscriptionId=nj&criterion=select%20country%20from%20stream%20where%20status%3D%3D500&clientId=nj2" 40 | * Here the user is submitted an MQL query select country from stream where status==500. 41 | */ 42 | public class SyntheticSourceJob extends MantisJobProvider { 43 | 44 | @Override 45 | public Job getJobInstance() { 46 | 47 | return 48 | MantisJob 49 | // synthetic source generates random RequestEvents. 50 | .source(new SyntheticSource()) 51 | // Tags events with queries that match 52 | .stage(new TaggingStage(), TaggingStage.config()) 53 | // A custom sink that processes query parameters to register and deregister MQL queries 54 | .sink(new TaggedDataSourceSink(new QueryRequestPreProcessor(), new QueryRequestPostProcessor())) 55 | // required parameters 56 | .create(); 57 | } 58 | 59 | public static void main(String[] args) { 60 | LocalJobExecutorNetworked.execute(new SyntheticSourceJob().getJobInstance()); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/java/io/mantisrx/sourcejob/synthetic/core/MQLQueryManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.sourcejob.synthetic.core; 18 | 19 | import java.util.Collection; 20 | import java.util.concurrent.ConcurrentHashMap; 21 | 22 | import io.mantisrx.mql.jvm.core.Query; 23 | 24 | 25 | public class MQLQueryManager { 26 | 27 | static class LazyHolder { 28 | 29 | private static final MQLQueryManager INSTANCE = new MQLQueryManager(); 30 | } 31 | 32 | private ConcurrentHashMap queries = new ConcurrentHashMap<>(); 33 | 34 | public static MQLQueryManager getInstance() { 35 | return LazyHolder.INSTANCE; 36 | } 37 | 38 | private MQLQueryManager() { } 39 | 40 | public void registerQuery(String id, String query) { 41 | query = MQL.transformLegacyQuery(query); 42 | Query q = MQL.makeQuery(id, query); 43 | queries.put(id, q); 44 | } 45 | 46 | public void deregisterQuery(String id) { 47 | queries.remove(id); 48 | } 49 | 50 | public Collection getRegisteredQueries() { 51 | return queries.values(); 52 | } 53 | 54 | public void clear() { 55 | queries.clear(); 56 | } 57 | 58 | public static void main(String[] args) throws Exception { 59 | MQLQueryManager qm = getInstance(); 60 | String query = "SELECT * WHERE true SAMPLE {\"strategy\":\"RANDOM\",\"threshold\":1}"; 61 | qm.registerQuery("fake2", query); 62 | System.out.println(MQL.parses(MQL.transformLegacyQuery(query))); 63 | System.out.println(MQL.getParseError(MQL.transformLegacyQuery(query))); 64 | System.out.println(qm.getRegisteredQueries()); 65 | } 66 | } 67 | 68 | 69 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/java/io/mantisrx/sourcejob/synthetic/core/TaggedData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.sourcejob.synthetic.core; 18 | 19 | import java.util.HashMap; 20 | import java.util.HashSet; 21 | import java.util.Map; 22 | import java.util.Set; 23 | 24 | import com.fasterxml.jackson.annotation.JsonCreator; 25 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 26 | import com.fasterxml.jackson.annotation.JsonProperty; 27 | import io.mantisrx.common.codec.Codec; 28 | import io.mantisrx.runtime.codec.JsonType; 29 | 30 | public class TaggedData implements JsonType { 31 | 32 | private final Set matchedClients = new HashSet(); 33 | private Map payLoad; 34 | 35 | @JsonCreator 36 | @JsonIgnoreProperties(ignoreUnknown = true) 37 | public TaggedData(@JsonProperty("data") Map data) { 38 | this.payLoad = data; 39 | } 40 | 41 | public Set getMatchedClients() { 42 | return matchedClients; 43 | } 44 | 45 | public boolean matchesClient(String clientId) { 46 | return matchedClients.contains(clientId); 47 | } 48 | 49 | public void addMatchedClient(String clientId) { 50 | matchedClients.add(clientId); 51 | } 52 | 53 | public Map getPayload() { 54 | return this.payLoad; 55 | } 56 | 57 | public void setPayload(Map newPayload) { 58 | this.payLoad = newPayload; 59 | } 60 | 61 | 62 | public static Codec taggedDataCodec() { 63 | 64 | return new Codec() { 65 | @Override 66 | public TaggedData decode(byte[] bytes) { 67 | return new TaggedData(new HashMap<>()); 68 | } 69 | 70 | @Override 71 | public byte[] encode(final TaggedData value) { 72 | return new byte[128]; 73 | } 74 | }; 75 | } 76 | 77 | 78 | 79 | } 80 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/java/io/mantisrx/sourcejob/synthetic/proto/RequestEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.sourcejob.synthetic.proto; 18 | 19 | import java.io.IOException; 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | import com.fasterxml.jackson.core.JsonProcessingException; 24 | import com.fasterxml.jackson.databind.ObjectMapper; 25 | import com.fasterxml.jackson.databind.ObjectReader; 26 | import io.mantisrx.common.codec.Codec; 27 | import lombok.Builder; 28 | import lombok.Data; 29 | 30 | 31 | /** 32 | * Represents a Request Event a service may receive. 33 | */ 34 | @Data 35 | @Builder 36 | public class RequestEvent { 37 | 38 | private static final ObjectMapper mapper = new ObjectMapper(); 39 | private static final ObjectReader requestEventReader = mapper.readerFor(RequestEvent.class); 40 | 41 | private final String userId; 42 | private final String uri; 43 | private final int status; 44 | private final String country; 45 | private final String deviceType; 46 | 47 | public Map toMap() { 48 | Map data = new HashMap<>(); 49 | data.put("userId", userId); 50 | data.put("uri", uri); 51 | data.put("status", status); 52 | data.put("country", country); 53 | data.put("deviceType", deviceType); 54 | return data; 55 | } 56 | 57 | public String toJsonString() { 58 | try { 59 | return mapper.writeValueAsString(this); 60 | } catch (JsonProcessingException e) { 61 | e.printStackTrace(); 62 | return null; 63 | } 64 | } 65 | 66 | /** 67 | * The codec defines how this class should be serialized before transporting across network. 68 | * @return 69 | */ 70 | public static Codec requestEventCodec() { 71 | 72 | return new Codec() { 73 | @Override 74 | public RequestEvent decode(byte[] bytes) { 75 | 76 | try { 77 | return requestEventReader.readValue(bytes); 78 | } catch (IOException e) { 79 | throw new RuntimeException(e); 80 | } 81 | } 82 | 83 | @Override 84 | public byte[] encode(final RequestEvent value) { 85 | 86 | try { 87 | return mapper.writeValueAsBytes(value); 88 | } catch (Exception e) { 89 | throw new RuntimeException(e); 90 | } 91 | } 92 | }; 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/java/io/mantisrx/sourcejob/synthetic/sink/QueryRefCountMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.sourcejob.synthetic.sink; 18 | 19 | import java.util.concurrent.ConcurrentHashMap; 20 | import java.util.concurrent.atomic.AtomicInteger; 21 | 22 | import io.mantisrx.sourcejob.synthetic.core.MQLQueryManager; 23 | import lombok.extern.slf4j.Slf4j; 24 | 25 | 26 | /** 27 | * This class keeps track of number of clients that have the exact same query registered for 28 | * deduplication purposes. 29 | * When all references to a query are gone the query is deregistered. 30 | */ 31 | @Slf4j 32 | final class QueryRefCountMap { 33 | 34 | public static final QueryRefCountMap INSTANCE = new QueryRefCountMap(); 35 | private final ConcurrentHashMap refCntMap = new ConcurrentHashMap<>(); 36 | 37 | private QueryRefCountMap() { } 38 | 39 | void addQuery(String subId, String query) { 40 | log.info("adding query " + subId + " query " + query); 41 | if (refCntMap.containsKey(subId)) { 42 | int newVal = refCntMap.get(subId).incrementAndGet(); 43 | log.info("query exists already incrementing refcnt to " + newVal); 44 | } else { 45 | MQLQueryManager.getInstance().registerQuery(subId, query); 46 | refCntMap.putIfAbsent(subId, new AtomicInteger(1)); 47 | log.info("new query registering it"); 48 | } 49 | } 50 | 51 | void removeQuery(String subId) { 52 | if (refCntMap.containsKey(subId)) { 53 | AtomicInteger refCnt = refCntMap.get(subId); 54 | int currVal = refCnt.decrementAndGet(); 55 | 56 | if (currVal == 0) { 57 | MQLQueryManager.getInstance().deregisterQuery(subId); 58 | refCntMap.remove(subId); 59 | log.info("All references to query are gone removing query"); 60 | } else { 61 | log.info("References to query still exist. decrementing refcnt to " + currVal); 62 | } 63 | } else { 64 | log.warn("No query with subscriptionId " + subId); 65 | } 66 | } 67 | 68 | /** 69 | * For testing 70 | * 71 | * @param subId 72 | * 73 | * @return 74 | */ 75 | int getQueryRefCount(String subId) { 76 | if (refCntMap.containsKey(subId)) { 77 | return refCntMap.get(subId).get(); 78 | } else { 79 | return 0; 80 | } 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/java/io/mantisrx/sourcejob/synthetic/sink/QueryRequestPostProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.sourcejob.synthetic.sink; 18 | 19 | 20 | import static com.mantisrx.common.utils.MantisSourceJobConstants.CRITERION_PARAM_NAME; 21 | import static com.mantisrx.common.utils.MantisSourceJobConstants.SUBSCRIPTION_ID_PARAM_NAME; 22 | 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | import io.mantisrx.runtime.Context; 27 | import lombok.extern.slf4j.Slf4j; 28 | import org.apache.log4j.Logger; 29 | import rx.functions.Func2; 30 | 31 | 32 | /** 33 | * This is a callback that is invoked after a client connected to the sink of this job disconnects. This is used 34 | * to cleanup the queries the client had registered. 35 | */ 36 | @Slf4j 37 | public class QueryRequestPostProcessor implements Func2>, Context, Void> { 38 | 39 | public QueryRequestPostProcessor() { } 40 | 41 | @Override 42 | public Void call(Map> queryParams, Context context) { 43 | log.info("RequestPostProcessor:queryParams: " + queryParams); 44 | 45 | if (queryParams != null) { 46 | 47 | if (queryParams.containsKey(SUBSCRIPTION_ID_PARAM_NAME) && queryParams.containsKey(CRITERION_PARAM_NAME)) { 48 | final String subId = queryParams.get(SUBSCRIPTION_ID_PARAM_NAME).get(0); 49 | final String query = queryParams.get(CRITERION_PARAM_NAME).get(0); 50 | final String clientId = queryParams.get("clientId").get(0); 51 | 52 | if (subId != null && query != null) { 53 | try { 54 | if (clientId != null && !clientId.isEmpty()) { 55 | deregisterQuery(clientId + "_" + subId); 56 | } else { 57 | deregisterQuery(subId); 58 | } 59 | } catch (Throwable t) { 60 | log.error("Error propagating unsubscription notification", t); 61 | } 62 | } 63 | } 64 | } 65 | return null; 66 | } 67 | 68 | private void deregisterQuery(String subId) { 69 | QueryRefCountMap.INSTANCE.removeQuery(subId); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/java/io/mantisrx/sourcejob/synthetic/sink/QueryRequestPreProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.sourcejob.synthetic.sink; 18 | 19 | import static com.mantisrx.common.utils.MantisSourceJobConstants.CRITERION_PARAM_NAME; 20 | import static com.mantisrx.common.utils.MantisSourceJobConstants.SUBSCRIPTION_ID_PARAM_NAME; 21 | 22 | import java.util.List; 23 | import java.util.Map; 24 | 25 | import io.mantisrx.runtime.Context; 26 | import lombok.extern.slf4j.Slf4j; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | import rx.functions.Func2; 30 | 31 | 32 | /** 33 | * This is a callback that is invoked when a new client connects to this sink of this job. 34 | * The callback is used to extract useful query parameters the user may have set on the GET request such as 35 | * the clientId, subscriptionId and the criterion. 36 | * The clientId identifies a group of connections belonging to the same consumer. Data is sent round-robin amongst 37 | * all clients with the same clientId 38 | * The subscriptionId tracks this particular client. 39 | * The criterion is a valid MQL query. It indicates what data this client is interested in. 40 | */ 41 | @Slf4j 42 | public class QueryRequestPreProcessor implements Func2>, Context, Void> { 43 | 44 | public QueryRequestPreProcessor() { } 45 | 46 | @Override 47 | public Void call(Map> queryParams, Context context) { 48 | 49 | log.info("QueryRequestPreProcessor:queryParams: {}", queryParams); 50 | 51 | if (queryParams != null) { 52 | 53 | if (queryParams.containsKey(SUBSCRIPTION_ID_PARAM_NAME) && queryParams.containsKey(CRITERION_PARAM_NAME)) { 54 | final String subId = queryParams.get(SUBSCRIPTION_ID_PARAM_NAME).get(0); 55 | final String query = queryParams.get(CRITERION_PARAM_NAME).get(0); 56 | final String clientId = queryParams.get("clientId").get(0); 57 | 58 | if (subId != null && query != null) { 59 | try { 60 | log.info("Registering query {}", query); 61 | if (clientId != null && !clientId.isEmpty()) { 62 | registerQuery(clientId + "_" + subId, query); 63 | } else { 64 | registerQuery(subId, query); 65 | } 66 | 67 | } catch (Throwable t) { 68 | log.error("Error registering query", t); 69 | } 70 | } 71 | } 72 | } 73 | return null; 74 | } 75 | 76 | private static synchronized void registerQuery(String subId, String query) { 77 | QueryRefCountMap.INSTANCE.addQuery(subId, query); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/java/io/mantisrx/sourcejob/synthetic/sink/TaggedDataSourceSink.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.sourcejob.synthetic.sink; 18 | 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | import com.fasterxml.jackson.core.JsonProcessingException; 23 | import com.fasterxml.jackson.databind.ObjectMapper; 24 | import io.mantisrx.runtime.Context; 25 | import io.mantisrx.runtime.PortRequest; 26 | import io.mantisrx.runtime.sink.ServerSentEventsSink; 27 | import io.mantisrx.runtime.sink.Sink; 28 | import io.mantisrx.runtime.sink.predicate.Predicate; 29 | import io.mantisrx.sourcejob.synthetic.core.TaggedData; 30 | import lombok.extern.slf4j.Slf4j; 31 | import rx.Observable; 32 | import rx.functions.Func2; 33 | 34 | 35 | /** 36 | * A custom sink that allows clients to connect to this job with an MQL expression and in turn receive events 37 | * matching this expression. 38 | */ 39 | @Slf4j 40 | public class TaggedDataSourceSink implements Sink { 41 | 42 | private Func2>, Context, Void> preProcessor = new NoOpProcessor(); 43 | private Func2>, Context, Void> postProcessor = new NoOpProcessor(); 44 | private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); 45 | 46 | static class NoOpProcessor implements Func2>, Context, Void> { 47 | 48 | @Override 49 | public Void call(Map> t1, Context t2) { 50 | return null; 51 | } 52 | } 53 | 54 | public TaggedDataSourceSink() { 55 | } 56 | 57 | public TaggedDataSourceSink(Func2>, Context, Void> preProcessor, 58 | Func2>, Context, Void> postProcessor) { 59 | this.postProcessor = postProcessor; 60 | this.preProcessor = preProcessor; 61 | } 62 | 63 | @Override 64 | public void call(Context context, PortRequest portRequest, 65 | Observable observable) { 66 | observable = observable 67 | .filter((t1) -> !t1.getPayload().isEmpty()); 68 | ServerSentEventsSink sink = new ServerSentEventsSink.Builder() 69 | .withEncoder((data) -> { 70 | try { 71 | return OBJECT_MAPPER.writeValueAsString(data.getPayload()); 72 | } catch (JsonProcessingException e) { 73 | e.printStackTrace(); 74 | return "{\"error\":" + e.getMessage() + "}"; 75 | } 76 | }) 77 | .withPredicate(new Predicate<>("description", new TaggedEventFilter())) 78 | .withRequestPreprocessor(preProcessor) 79 | .withRequestPostprocessor(postProcessor) 80 | .build(); 81 | 82 | observable.subscribe(); 83 | sink.call(context, portRequest, observable); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/java/io/mantisrx/sourcejob/synthetic/sink/TaggedEventFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.sourcejob.synthetic.sink; 18 | 19 | import static com.mantisrx.common.utils.MantisSourceJobConstants.CLIENT_ID_PARAMETER_NAME; 20 | import static com.mantisrx.common.utils.MantisSourceJobConstants.SUBSCRIPTION_ID_PARAM_NAME; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | import io.mantisrx.sourcejob.synthetic.core.TaggedData; 27 | import lombok.extern.slf4j.Slf4j; 28 | import rx.functions.Func1; 29 | 30 | 31 | /** 32 | * This is a predicate that decides what data to send to the downstream client. The data is tagged with the clientId 33 | * and subscriptionId of the intended recipient. 34 | */ 35 | @Slf4j 36 | public class TaggedEventFilter implements Func1>, Func1> { 37 | 38 | @Override 39 | public Func1 call(Map> parameters) { 40 | Func1 filter = t1 -> true; 41 | if (parameters != null) { 42 | if (parameters.containsKey(SUBSCRIPTION_ID_PARAM_NAME)) { 43 | String subId = parameters.get(SUBSCRIPTION_ID_PARAM_NAME).get(0); 44 | String clientId = parameters.get(CLIENT_ID_PARAMETER_NAME).get(0); 45 | List terms = new ArrayList(); 46 | if (clientId != null && !clientId.isEmpty()) { 47 | terms.add(clientId + "_" + subId); 48 | } else { 49 | terms.add(subId); 50 | } 51 | filter = new SourceEventFilter(terms); 52 | } 53 | return filter; 54 | } 55 | return filter; 56 | } 57 | 58 | private static class SourceEventFilter implements Func1 { 59 | 60 | private String jobId = "UNKNOWN"; 61 | private String jobName = "UNKNOWN"; 62 | private List terms; 63 | 64 | SourceEventFilter(List terms) { 65 | this.terms = terms; 66 | String jId = System.getenv("JOB_ID"); 67 | if (jId != null && !jId.isEmpty()) { 68 | jobId = jId; 69 | } 70 | String jName = System.getenv("JOB_NAME"); 71 | if (jName != null && !jName.isEmpty()) { 72 | jobName = jName; 73 | } 74 | log.info("Created SourceEventFilter! for subId " + terms.toString() + " in Job : " + jobName + " with Id " + jobId); 75 | } 76 | 77 | @Override 78 | public Boolean call(TaggedData data) { 79 | 80 | boolean match = true; 81 | for (String term : terms) { 82 | if (!data.matchesClient(term)) { 83 | match = false; 84 | break; 85 | } 86 | } 87 | return match; 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/java/io/mantisrx/sourcejob/synthetic/source/SyntheticSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.sourcejob.synthetic.source; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | import java.util.Objects; 22 | import java.util.concurrent.TimeUnit; 23 | 24 | import io.mantisrx.runtime.Context; 25 | import io.mantisrx.runtime.parameter.ParameterDefinition; 26 | import io.mantisrx.runtime.parameter.type.IntParameter; 27 | import io.mantisrx.runtime.parameter.validator.Validators; 28 | import io.mantisrx.runtime.source.Index; 29 | import io.mantisrx.runtime.source.Source; 30 | import io.mantisrx.sourcejob.synthetic.proto.RequestEvent; 31 | import lombok.extern.slf4j.Slf4j; 32 | import net.andreinc.mockneat.MockNeat; 33 | import rx.Observable; 34 | 35 | 36 | /** 37 | * Generates random set of RequestEvents at a preconfigured interval. 38 | */ 39 | @Slf4j 40 | public class SyntheticSource implements Source { 41 | 42 | private static final String DATA_GENERATION_RATE_MSEC_PARAM = "dataGenerationRate"; 43 | private MockNeat mockDataGenerator; 44 | private int dataGenerateRateMsec = 250; 45 | 46 | @Override 47 | public Observable> call(Context context, Index index) { 48 | return Observable.just(Observable 49 | .interval(dataGenerateRateMsec, TimeUnit.MILLISECONDS) 50 | .map((tick) -> generateEvent()) 51 | .map((event) -> event.toJsonString()) 52 | .filter(Objects::nonNull) 53 | .doOnNext((event) -> { 54 | log.debug("Generated Event {}", event); 55 | })); 56 | } 57 | 58 | @Override 59 | public void init(Context context, Index index) { 60 | mockDataGenerator = MockNeat.threadLocal(); 61 | dataGenerateRateMsec = (int)context.getParameters().get(DATA_GENERATION_RATE_MSEC_PARAM,250); 62 | } 63 | @Override 64 | public List> getParameters() { 65 | List> params = new ArrayList<>(); 66 | params.add(new IntParameter() 67 | .name(DATA_GENERATION_RATE_MSEC_PARAM) 68 | .description("Rate at which to generate data") 69 | .validator(Validators.range(100,1000000)) 70 | .defaultValue(250) 71 | .build()); 72 | 73 | return params; 74 | } 75 | 76 | private RequestEvent generateEvent() { 77 | 78 | String path = mockDataGenerator.probabilites(String.class) 79 | .add(0.1, "/login") 80 | .add(0.2, "/genre/horror") 81 | .add(0.5, "/genre/comedy") 82 | .add(0.2, "/mylist") 83 | .get(); 84 | 85 | String deviceType = mockDataGenerator.probabilites(String.class) 86 | .add(0.1, "ps4") 87 | .add(0.1, "xbox") 88 | .add(0.2, "browser") 89 | .add(0.3, "ios") 90 | .add(0.3, "android") 91 | .get(); 92 | String userId = mockDataGenerator.strings().size(10).get(); 93 | int status = mockDataGenerator.probabilites(Integer.class) 94 | .add(0.1,500) 95 | .add(0.7,200) 96 | .add(0.2,500) 97 | .get(); 98 | 99 | 100 | String country = mockDataGenerator.countries().names().get(); 101 | return RequestEvent.builder() 102 | .status(status) 103 | .uri(path) 104 | .country(country) 105 | .userId(userId) 106 | .deviceType(deviceType) 107 | .build(); 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/java/io/mantisrx/sourcejob/synthetic/stage/TaggingStage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.sourcejob.synthetic.stage; 18 | 19 | import java.util.ArrayList; 20 | import java.util.Collection; 21 | import java.util.HashMap; 22 | import java.util.Iterator; 23 | import java.util.List; 24 | import java.util.Map; 25 | import java.util.Objects; 26 | import java.util.concurrent.atomic.AtomicBoolean; 27 | 28 | import com.mantisrx.common.utils.JsonUtility; 29 | import io.mantisrx.common.codec.Codec; 30 | import io.mantisrx.common.metrics.Metrics; 31 | import io.mantisrx.common.metrics.spectator.MetricGroupId; 32 | import io.mantisrx.mql.jvm.core.Query; 33 | import io.mantisrx.runtime.Context; 34 | import io.mantisrx.runtime.ScalarToScalar; 35 | import io.mantisrx.runtime.computation.ScalarComputation; 36 | import io.mantisrx.sourcejob.synthetic.core.MQLQueryManager; 37 | import io.mantisrx.sourcejob.synthetic.core.TaggedData; 38 | import lombok.extern.slf4j.Slf4j; 39 | import rx.Observable; 40 | 41 | 42 | /** 43 | * Tags incoming events with ids of queries that evaluate to true against the data. 44 | * 45 | * Each event is first transformed into a Map, next each query from the list of Registered MQL queries 46 | * is applied against the event. The event is tagged with the ids of queries that matched. 47 | * 48 | */ 49 | @Slf4j 50 | public class TaggingStage implements ScalarComputation { 51 | 52 | public static final String MANTIS_META_SOURCE_NAME = "mantis.meta.sourceName"; 53 | public static final String MANTIS_META_SOURCE_TIMESTAMP = "mantis.meta.timestamp"; 54 | public static final String MANTIS_QUERY_COUNTER = "mantis_query_out"; 55 | public static final String MQL_COUNTER = "mql_out"; 56 | public static final String MQL_FAILURE = "mql_failure"; 57 | public static final String MQL_CLASSLOADER_ERROR = "mql_classloader_error"; 58 | public static final String SYNTHETIC_REQUEST_SOURCE = "SyntheticRequestSource"; 59 | 60 | private AtomicBoolean errorLogged = new AtomicBoolean(false); 61 | @Override 62 | public Observable call(Context context, Observable dataO) { 63 | return dataO 64 | .map((event) -> { 65 | try { 66 | return JsonUtility.jsonToMap(event); 67 | } catch (Exception e) { 68 | log.error(e.getMessage()); 69 | return null; 70 | } 71 | }) 72 | .filter(Objects::nonNull) 73 | .flatMapIterable(d -> tagData(d, context)); 74 | } 75 | 76 | @Override 77 | public void init(Context context) { 78 | 79 | context.getMetricsRegistry().registerAndGet(new Metrics.Builder() 80 | .name("mql") 81 | .addCounter(MQL_COUNTER) 82 | .addCounter(MQL_FAILURE) 83 | .addCounter(MQL_CLASSLOADER_ERROR) 84 | .addCounter(MANTIS_QUERY_COUNTER).build()); 85 | } 86 | 87 | 88 | private List tagData(Map d, Context context) { 89 | List taggedDataList = new ArrayList<>(); 90 | 91 | Metrics metrics = context.getMetricsRegistry().getMetric(new MetricGroupId("mql")); 92 | 93 | Collection queries = MQLQueryManager.getInstance().getRegisteredQueries(); 94 | Iterator it = queries.iterator(); 95 | while (it.hasNext()) { 96 | Query query = it.next(); 97 | try { 98 | if (query.matches(d)) { 99 | Map projected = query.project(d); 100 | projected.put(MANTIS_META_SOURCE_NAME, SYNTHETIC_REQUEST_SOURCE); 101 | projected.put(MANTIS_META_SOURCE_TIMESTAMP, System.currentTimeMillis()); 102 | TaggedData tg = new TaggedData(projected); 103 | tg.addMatchedClient(query.getSubscriptionId()); 104 | taggedDataList.add(tg); 105 | } 106 | } catch (Exception ex) { 107 | if (ex instanceof ClassNotFoundException) { 108 | log.error("Error loading MQL: " + ex.getMessage()); 109 | ex.printStackTrace(); 110 | metrics.getCounter(MQL_CLASSLOADER_ERROR).increment(); 111 | } else { 112 | ex.printStackTrace(); 113 | metrics.getCounter(MQL_FAILURE).increment(); 114 | log.error("MQL Error: " + ex.getMessage()); 115 | log.error("MQL Query: " + query.getRawQuery()); 116 | log.error("MQL Datum: " + d); 117 | } 118 | } catch (Error e) { 119 | metrics.getCounter(MQL_FAILURE).increment(); 120 | if (!errorLogged.get()) { 121 | log.error("caught Error when processing MQL {} on {}", query.getRawQuery(), d.toString(), e); 122 | errorLogged.set(true); 123 | } 124 | } 125 | } 126 | 127 | return taggedDataList; 128 | } 129 | 130 | public static ScalarToScalar.Config config() { 131 | return new ScalarToScalar.Config() 132 | .concurrentInput() 133 | .codec(TaggingStage.taggedDataCodec()); 134 | } 135 | 136 | public static Codec taggedDataCodec() { 137 | 138 | return new Codec() { 139 | @Override 140 | public TaggedData decode(byte[] bytes) { 141 | return new TaggedData(new HashMap<>()); 142 | } 143 | 144 | @Override 145 | public byte[] encode(final TaggedData value) { 146 | return new byte[128]; 147 | } 148 | }; 149 | } 150 | 151 | 152 | 153 | 154 | } 155 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/resources/META-INF/services/io.mantisrx.runtime.MantisJobProvider: -------------------------------------------------------------------------------- 1 | io.mantisrx.sourcejob.synthetic.SyntheticSourceJob 2 | -------------------------------------------------------------------------------- /synthetic-sourcejob/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /twitter-sample/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | 3 | configurations.all { 4 | resolutionStrategy { 5 | force "com.google.guava:guava:18.0" 6 | force "org.apache.httpcomponents:httpclient:4.5.9" 7 | } 8 | } 9 | 10 | 11 | task execute(type:JavaExec) { 12 | main = "com.netflix.mantis.examples.twittersample.TwitterJob" 13 | classpath = sourceSets.main.runtimeClasspath 14 | } 15 | 16 | dependencies { 17 | compile 'com.twitter:hbc-core:2.2.0' 18 | compile 'io.mantisrx:mantis-runtime:1.2.+' 19 | 20 | compile 'org.slf4j:slf4j-api' 21 | compile 'org.slf4j:slf4j-log4j12' 22 | 23 | compileOnly "org.projectlombok:lombok:1.16.16" 24 | } 25 | 26 | 27 | -------------------------------------------------------------------------------- /twitter-sample/src/main/java/com/netflix/mantis/examples/config/StageConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.config; 18 | 19 | import java.util.Map; 20 | 21 | import io.mantisrx.common.codec.Codecs; 22 | import io.mantisrx.runtime.KeyToScalar; 23 | import io.mantisrx.runtime.ScalarToKey; 24 | import io.mantisrx.runtime.ScalarToScalar; 25 | import io.mantisrx.runtime.codec.JacksonCodecs; 26 | 27 | 28 | public class StageConfigs { 29 | 30 | public static ScalarToScalar.Config scalarToScalarConfig() { 31 | return new ScalarToScalar.Config() 32 | .codec(Codecs.string()); 33 | } 34 | 35 | public static KeyToScalar.Config, String> keyToScalarConfig() { 36 | return new KeyToScalar.Config, String>() 37 | .description("sum events ") 38 | .keyExpireTimeSeconds(10) 39 | .codec(Codecs.string()); 40 | } 41 | 42 | public static ScalarToKey.Config> scalarToKeyConfig() { 43 | return new ScalarToKey.Config>() 44 | .description("Group event data by ip") 45 | .concurrentInput() 46 | .keyExpireTimeSeconds(1) 47 | .codec(JacksonCodecs.mapStringObject()); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /twitter-sample/src/main/java/com/netflix/mantis/examples/core/ObservableQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.core; 18 | 19 | import java.io.Closeable; 20 | import java.io.IOException; 21 | import java.util.Collection; 22 | import java.util.Collections; 23 | import java.util.Iterator; 24 | import java.util.NoSuchElementException; 25 | import java.util.concurrent.BlockingQueue; 26 | import java.util.concurrent.TimeUnit; 27 | 28 | import rx.Observable; 29 | import rx.subjects.PublishSubject; 30 | import rx.subjects.Subject; 31 | 32 | 33 | /** 34 | * An Observable that acts as a blocking queue. It is backed by a Subject 35 | * 36 | * @param 37 | */ 38 | public class ObservableQueue implements BlockingQueue, Closeable { 39 | 40 | private final Subject subject = PublishSubject.create().toSerialized(); 41 | 42 | public Observable observe() { 43 | return subject; 44 | } 45 | 46 | @Override 47 | public boolean add(T t) { 48 | return offer(t); 49 | } 50 | 51 | @Override 52 | public boolean offer(T t) { 53 | subject.onNext(t); 54 | return true; 55 | } 56 | 57 | @Override 58 | public void close() throws IOException { 59 | subject.onCompleted(); 60 | } 61 | 62 | @Override 63 | public T remove() { 64 | return noSuchElement(); 65 | } 66 | 67 | @Override 68 | public T poll() { 69 | return null; 70 | } 71 | 72 | @Override 73 | public T element() { 74 | return noSuchElement(); 75 | } 76 | 77 | private T noSuchElement() { 78 | throw new NoSuchElementException(); 79 | } 80 | 81 | @Override 82 | public T peek() { 83 | return null; 84 | } 85 | 86 | @Override 87 | public void put(T t) throws InterruptedException { 88 | offer(t); 89 | } 90 | 91 | @Override 92 | public boolean offer(T t, long timeout, TimeUnit unit) throws InterruptedException { 93 | return offer(t); 94 | } 95 | 96 | @Override 97 | public T take() throws InterruptedException { 98 | throw new UnsupportedOperationException("Use observe() instead"); 99 | } 100 | 101 | @Override 102 | public T poll(long timeout, TimeUnit unit) throws InterruptedException { 103 | return null; 104 | } 105 | 106 | @Override 107 | public int remainingCapacity() { 108 | return 0; 109 | } 110 | 111 | @Override 112 | public boolean remove(Object o) { 113 | return false; 114 | } 115 | 116 | @Override 117 | public boolean containsAll(Collection c) { 118 | return false; 119 | } 120 | 121 | @Override 122 | public boolean addAll(Collection c) { 123 | c.forEach(this::offer); 124 | return true; 125 | } 126 | 127 | @Override 128 | public boolean removeAll(Collection c) { 129 | return false; 130 | } 131 | 132 | @Override 133 | public boolean retainAll(Collection c) { 134 | return false; 135 | } 136 | 137 | @Override 138 | public void clear() { 139 | } 140 | 141 | @Override 142 | public int size() { 143 | return 0; 144 | } 145 | 146 | @Override 147 | public boolean isEmpty() { 148 | return true; 149 | } 150 | 151 | @Override 152 | public boolean contains(Object o) { 153 | return false; 154 | } 155 | 156 | @Override 157 | public Iterator iterator() { 158 | return Collections.emptyIterator(); 159 | } 160 | 161 | @Override 162 | public Object[] toArray() { 163 | return new Object[0]; 164 | } 165 | 166 | @Override 167 | public T[] toArray(T[] a) { 168 | return a; 169 | } 170 | 171 | @Override 172 | public int drainTo(Collection c) { 173 | return 0; 174 | } 175 | 176 | @Override 177 | public int drainTo(Collection c, int maxElements) { 178 | return 0; 179 | } 180 | 181 | } -------------------------------------------------------------------------------- /twitter-sample/src/main/java/com/netflix/mantis/examples/core/WordCountPair.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.core; 18 | 19 | import lombok.Data; 20 | 21 | 22 | /** 23 | * A simple class that holds a word and a count of how many times it has occurred. 24 | */ 25 | @Data 26 | public class WordCountPair { 27 | 28 | private final String word; 29 | private final int count; 30 | } 31 | -------------------------------------------------------------------------------- /twitter-sample/src/main/java/com/netflix/mantis/examples/wordcount/TwitterJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.wordcount; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | import java.util.StringTokenizer; 22 | import java.util.concurrent.TimeUnit; 23 | 24 | import com.mantisrx.common.utils.JsonUtility; 25 | import com.netflix.mantis.examples.config.StageConfigs; 26 | import com.netflix.mantis.examples.core.WordCountPair; 27 | import com.netflix.mantis.examples.wordcount.sources.TwitterSource; 28 | import io.mantisrx.runtime.Job; 29 | import io.mantisrx.runtime.MantisJob; 30 | import io.mantisrx.runtime.MantisJobProvider; 31 | import io.mantisrx.runtime.Metadata; 32 | import io.mantisrx.runtime.executor.LocalJobExecutorNetworked; 33 | import io.mantisrx.runtime.parameter.Parameter; 34 | import io.mantisrx.runtime.sink.Sinks; 35 | import lombok.extern.slf4j.Slf4j; 36 | import rx.Observable; 37 | 38 | 39 | /** 40 | * This sample demonstrates connecting to a twitter feed and counting the number of occurrences of words within a 10 41 | * sec hopping window. 42 | * Run the main method of this class and then look for a the SSE port in the output 43 | * E.g 44 | * Serving modern HTTP SSE server sink on port: 8650 45 | * You can curl this port curl localhost:8650 to view the output of the job. 46 | * 47 | * To run via gradle 48 | * ../gradlew execute --args='consumerKey consumerSecret token tokensecret' 49 | */ 50 | @Slf4j 51 | public class TwitterJob extends MantisJobProvider { 52 | 53 | @Override 54 | public Job getJobInstance() { 55 | return MantisJob 56 | .source(new TwitterSource()) 57 | // Simply echoes the tweet 58 | .stage((context, dataO) -> dataO 59 | .map(JsonUtility::jsonToMap) 60 | // filter out english tweets 61 | .filter((eventMap) -> { 62 | if(eventMap.containsKey("lang") && eventMap.containsKey("text")) { 63 | String lang = (String)eventMap.get("lang"); 64 | return "en".equalsIgnoreCase(lang); 65 | } 66 | return false; 67 | }).map((eventMap) -> (String)eventMap.get("text")) 68 | // tokenize the tweets into words 69 | .flatMap((text) -> Observable.from(tokenize(text))) 70 | // On a hopping window of 10 seconds 71 | .window(10, TimeUnit.SECONDS) 72 | .flatMap((wordCountPairObservable) -> wordCountPairObservable 73 | // count how many times a word appears 74 | .groupBy(WordCountPair::getWord) 75 | .flatMap((groupO) -> groupO.reduce(0, (cnt, wordCntPair) -> cnt + 1) 76 | .map((cnt) -> new WordCountPair(groupO.getKey(), cnt)))) 77 | .map(WordCountPair::toString) 78 | .doOnNext((cnt) -> log.info(cnt)) 79 | , StageConfigs.scalarToScalarConfig()) 80 | // Reuse built in sink that eagerly subscribes and delivers data over SSE 81 | .sink(Sinks.eagerSubscribe(Sinks.sse((String data) -> data))) 82 | .metadata(new Metadata.Builder() 83 | .name("TwitterSample") 84 | .description("Connects to a Twitter feed") 85 | .build()) 86 | .create(); 87 | } 88 | 89 | private List tokenize(String text) { 90 | StringTokenizer tokenizer = new StringTokenizer(text); 91 | List wordCountPairs = new ArrayList<>(); 92 | while(tokenizer.hasMoreTokens()) { 93 | String word = tokenizer.nextToken().replaceAll("\\s*", "").toLowerCase(); 94 | wordCountPairs.add(new WordCountPair(word,1)); 95 | } 96 | return wordCountPairs; 97 | } 98 | 99 | 100 | public static void main(String[] args) { 101 | 102 | String consumerKey = null; 103 | String consumerSecret = null; 104 | String token = null; 105 | String tokenSecret = null; 106 | if(args.length != 4) { 107 | System.out.println("Usage: java com.netflix.mantis.examples.TwitterJob { 44 | 45 | public static final String CONSUMER_KEY_PARAM = "consumerKey"; 46 | 47 | public static final String CONSUMER_SECRET_PARAM = "consumerSecret"; 48 | 49 | public static final String TOKEN_PARAM = "token"; 50 | 51 | public static final String TOKEN_SECRET_PARAM = "tokenSecret"; 52 | 53 | public static final String TERMS_PARAM = "terms"; 54 | 55 | private final ObservableQueue twitterObservable = new ObservableQueue<>(); 56 | 57 | private transient BasicClient client; 58 | 59 | @Override 60 | public Observable> call(Context context, Index index) { 61 | return Observable.just(twitterObservable.observe()); 62 | } 63 | 64 | /** 65 | * Define parameters required by this source. 66 | * 67 | * @return 68 | */ 69 | @Override 70 | public List> getParameters() { 71 | List> params = Lists.newArrayList(); 72 | 73 | // Consumer key 74 | params.add(new StringParameter() 75 | .name(CONSUMER_KEY_PARAM) 76 | .description("twitter consumer key") 77 | .validator(Validators.notNullOrEmpty()) 78 | .required() 79 | .build()); 80 | 81 | params.add(new StringParameter() 82 | .name(CONSUMER_SECRET_PARAM) 83 | .description("twitter consumer secret") 84 | .validator(Validators.notNullOrEmpty()) 85 | .required() 86 | .build()); 87 | 88 | params.add(new StringParameter() 89 | .name(TOKEN_PARAM) 90 | .description("twitter token") 91 | .validator(Validators.notNullOrEmpty()) 92 | .required() 93 | .build()); 94 | 95 | params.add(new StringParameter() 96 | .name(TOKEN_SECRET_PARAM) 97 | .description("twitter token secret") 98 | .validator(Validators.notNullOrEmpty()) 99 | .required() 100 | .build()); 101 | 102 | params.add(new StringParameter() 103 | .name(TERMS_PARAM) 104 | .description("terms to follow") 105 | .validator(Validators.notNullOrEmpty()) 106 | .defaultValue("Netflix,Dark") 107 | .build()); 108 | 109 | return params; 110 | 111 | } 112 | 113 | /** 114 | * Init method is called only once during initialization. It is the ideal place to perform one time 115 | * configuration actions. 116 | * 117 | * @param context Provides access to Mantis system information like JobId, Job parameters etc 118 | * @param index This provides access to the unique workerIndex assigned to this container. It also provides 119 | * the total number of workers of this job. 120 | */ 121 | @Override 122 | public void init(Context context, Index index) { 123 | String consumerKey = (String) context.getParameters().get(CONSUMER_KEY_PARAM); 124 | String consumerSecret = (String) context.getParameters().get(CONSUMER_SECRET_PARAM); 125 | String token = (String) context.getParameters().get(TOKEN_PARAM); 126 | String tokenSecret = (String) context.getParameters().get(TOKEN_SECRET_PARAM); 127 | 128 | String terms = (String) context.getParameters().get(TERMS_PARAM); 129 | 130 | Authentication auth = new OAuth1(consumerKey, 131 | consumerSecret, 132 | token, 133 | tokenSecret); 134 | 135 | StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint(); 136 | 137 | String[] termArray = terms.split(","); 138 | 139 | List termsList = Arrays.asList(termArray); 140 | 141 | endpoint.trackTerms(termsList); 142 | 143 | client = new ClientBuilder() 144 | .name("twitter-source") 145 | .hosts(Constants.STREAM_HOST) 146 | .endpoint(endpoint) 147 | .authentication(auth) 148 | .processor(new StringDelimitedProcessor(twitterObservable)) 149 | .build(); 150 | 151 | client.connect(); 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /twitter-sample/src/main/resources/META-INF/services/io.mantisrx.runtime.MantisJobProvider: -------------------------------------------------------------------------------- 1 | com.netflix.mantis.examples.wordcount.WordCountJob -------------------------------------------------------------------------------- /twitter-sample/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /wordcount/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | 3 | configurations.all { 4 | resolutionStrategy { 5 | force "com.google.guava:guava:18.0" 6 | force "org.apache.httpcomponents:httpclient:4.5.9" 7 | } 8 | } 9 | 10 | 11 | task execute(type:JavaExec) { 12 | main = "com.netflix.mantis.examples.wordcount.WordCountJob" 13 | classpath = sourceSets.main.runtimeClasspath 14 | } 15 | 16 | dependencies { 17 | compile project(':mantis-examples-core') 18 | compile 'com.twitter:hbc-core:2.2.0' 19 | compile 'io.mantisrx:mantis-runtime:1.2.+' 20 | 21 | compile 'org.slf4j:slf4j-api' 22 | compile 'org.slf4j:slf4j-log4j12' 23 | 24 | compileOnly "org.projectlombok:lombok:1.16.16" 25 | } 26 | 27 | 28 | -------------------------------------------------------------------------------- /wordcount/src/main/java/com/netflix/mantis/examples/wordcount/WordCountJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.wordcount; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | import java.util.StringTokenizer; 22 | import java.util.concurrent.TimeUnit; 23 | 24 | import com.netflix.mantis.examples.config.StageConfigs; 25 | import com.netflix.mantis.examples.core.WordCountPair; 26 | import com.netflix.mantis.examples.wordcount.sources.IlliadSource; 27 | import io.mantisrx.runtime.Job; 28 | import io.mantisrx.runtime.MantisJob; 29 | import io.mantisrx.runtime.MantisJobProvider; 30 | import io.mantisrx.runtime.Metadata; 31 | import io.mantisrx.runtime.executor.LocalJobExecutorNetworked; 32 | import io.mantisrx.runtime.sink.Sinks; 33 | import lombok.extern.slf4j.Slf4j; 34 | import rx.Observable; 35 | 36 | 37 | /** 38 | * This sample demonstrates ingesting data from a text file and counting the number of occurrences of words within a 10 39 | * sec hopping window. 40 | * Run the main method of this class and then look for a the SSE port in the output 41 | * E.g 42 | * Serving modern HTTP SSE server sink on port: 8650 43 | * You can curl this port curl localhost:8650 to view the output of the job. 44 | * 45 | * To run via gradle 46 | * /gradlew :mantis-examples-wordcount:execute 47 | */ 48 | @Slf4j 49 | public class WordCountJob extends MantisJobProvider { 50 | 51 | @Override 52 | public Job getJobInstance() { 53 | return MantisJob 54 | .source(new IlliadSource()) 55 | // Simply echoes the tweet 56 | .stage((context, dataO) -> dataO 57 | // Tokenize 58 | .flatMap((text) -> Observable.from(tokenize(text))) 59 | // On a hopping window of 10 seconds 60 | .window(10, TimeUnit.SECONDS) 61 | .flatMap((wordCountPairObservable) -> wordCountPairObservable 62 | // count how many times a word appears 63 | .groupBy(WordCountPair::getWord) 64 | .flatMap((groupO) -> groupO.reduce(0, (cnt, wordCntPair) -> cnt + 1) 65 | .map((cnt) -> new WordCountPair(groupO.getKey(), cnt)))) 66 | .map(WordCountPair::toString) 67 | , StageConfigs.scalarToScalarConfig()) 68 | // Reuse built in sink that eagerly subscribes and delivers data over SSE 69 | .sink(Sinks.eagerSubscribe(Sinks.sse((String data) -> data))) 70 | .metadata(new Metadata.Builder() 71 | .name("WordCount") 72 | .description("Reads Homer's The Illiad faster than we can.") 73 | .build()) 74 | .create(); 75 | } 76 | 77 | private List tokenize(String text) { 78 | StringTokenizer tokenizer = new StringTokenizer(text); 79 | List wordCountPairs = new ArrayList<>(); 80 | while(tokenizer.hasMoreTokens()) { 81 | String word = tokenizer.nextToken().replaceAll("\\s*", "").toLowerCase(); 82 | wordCountPairs.add(new WordCountPair(word,1)); 83 | } 84 | return wordCountPairs; 85 | } 86 | 87 | 88 | public static void main(String[] args) { 89 | LocalJobExecutorNetworked.execute(new WordCountJob().getJobInstance()); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /wordcount/src/main/java/com/netflix/mantis/examples/wordcount/sources/IlliadSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.netflix.mantis.examples.wordcount.sources; 18 | 19 | import java.io.IOException; 20 | import java.net.URISyntaxException; 21 | import java.nio.file.Files; 22 | import java.nio.file.Path; 23 | import java.nio.file.Paths; 24 | import java.util.concurrent.TimeUnit; 25 | import java.util.stream.Stream; 26 | 27 | import io.mantisrx.runtime.Context; 28 | import io.mantisrx.runtime.source.Index; 29 | import io.mantisrx.runtime.source.Source; 30 | import lombok.extern.log4j.Log4j; 31 | import rx.Observable; 32 | 33 | 34 | /** 35 | * Ignore the contents of this file for the tutorial. The purpose is just to generate a stream of interesting data 36 | * on which we can word count. 37 | */ 38 | @Log4j 39 | public class IlliadSource implements Source { 40 | 41 | @Override 42 | public Observable> call(Context context, Index index) { 43 | return Observable.interval(10, TimeUnit.SECONDS) 44 | .map(__ -> { 45 | try { 46 | Path path = Paths.get(getClass().getClassLoader() 47 | .getResource("illiad.txt").toURI()); 48 | return Observable.from(() -> { 49 | try { 50 | return Files.lines(path).iterator(); 51 | } catch (IOException ex) { 52 | log.error("IOException while reading illiad.txt from resources", ex); 53 | } 54 | return Stream.empty().iterator(); 55 | } 56 | ); 57 | } catch (URISyntaxException ex) { 58 | log.error("URISyntaxException while loading illiad.txt from resources.", ex); 59 | } 60 | return Observable.empty(); 61 | }); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /wordcount/src/main/resources/META-INF/services/io.mantisrx.runtime.MantisJobProvider: -------------------------------------------------------------------------------- 1 | com.netflix.mantis.examples.wordcount.WordCountJob -------------------------------------------------------------------------------- /wordcount/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n --------------------------------------------------------------------------------