├── .bookignore ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SUMMARY.md ├── book.json ├── build.sbt ├── devops ├── README.md ├── deploy-site.sh └── mac │ └── eigenflow ├── doc ├── error-handling.md ├── getting-started.md ├── messaging.md ├── process-stages.md ├── recovery.md └── time-management.md ├── project ├── build.properties └── plugins.sbt └── src ├── it ├── resources │ ├── application.conf │ └── logback.xml └── scala │ └── com.mediative.eigenflow.test.it │ ├── process │ ├── ProcessFSMTest.scala │ └── ProcessManagerTest.scala │ └── utils │ ├── EigenflowIntegrationTest.scala │ ├── models │ ├── FailingProcess.scala │ └── OneStageProcess.scala │ └── wrappers │ ├── TracedProcessFSM.scala │ └── TracedProcessManager.scala ├── main ├── resources │ ├── mesos.conf │ └── reference.conf └── scala │ └── com.mediative.eigenflow │ ├── EigenflowBootstrap.scala │ ├── StagedProcess.scala │ ├── domain │ ├── ProcessContext.scala │ ├── RecoveryStrategy.scala │ ├── RetryRegistry.scala │ ├── fsm │ │ ├── ExecutionPlan.scala │ │ ├── ProcessEvent.scala │ │ └── ProcessStage.scala │ └── messages │ │ ├── GenericMessage.scala │ │ ├── MetricsMessage.scala │ │ ├── ProcessMessage.scala │ │ ├── StageMessage.scala │ │ └── StateRecord.scala │ ├── dsl │ └── EigenflowDSL.scala │ ├── environment │ └── ConfigurationLoader.scala │ ├── helpers │ ├── DateHelper.scala │ └── PrimitiveImplicits.scala │ ├── process │ ├── ProcessFSM.scala │ ├── ProcessManager.scala │ └── ProcessSupervisor.scala │ └── publisher │ ├── MessagingSystem.scala │ ├── PrintMessagingSystem.scala │ ├── ProcessPublisher.scala │ └── kafka │ ├── KafkaConfiguration.scala │ └── KafkaMessagingSystem.scala └── test └── scala ├── com.mediative.eigenflow.test ├── dsl │ └── EigenflowDSLTest.scala ├── helpers │ └── DateHelperTest.scala ├── package.scala └── publisher │ └── ProcessPublisherTest.scala └── com └── mediative └── eigenflow └── publisher └── MessagingSystemTest.scala /.bookignore: -------------------------------------------------------------------------------- 1 | *.properties 2 | *.sbt 3 | *.sh 4 | *.yml 5 | LICENSE 6 | src/ 7 | project/ 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .idea/ 3 | node_modules/ 4 | _book/ 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Use container-based infrastructure 2 | sudo: false 3 | 4 | language: scala 5 | scala: 2.11.7 6 | jdk: oraclejdk8 7 | 8 | before_install: 9 | - npm set progress=false 10 | - npm install -g --prefix node_modules/ gitbook-cli 11 | - export PATH="$PATH:$PWD/node_modules/bin/" 12 | - gitbook install 13 | 14 | script: 15 | - sbt ++$TRAVIS_SCALA_VERSION reformat-code-check test it:test 16 | 17 | # Tricks to avoid unnecessary cache updates 18 | - find $HOME/.sbt -name "*.lock" | xargs rm 19 | - find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm 20 | 21 | before_deploy: 22 | - gitbook build 23 | 24 | deploy: 25 | provider: script 26 | script: devops/deploy-site.sh 27 | skip_cleanup: true 28 | on: 29 | branch: master 30 | condition: $TRAVIS_PULL_REQUEST = false 31 | 32 | # These directories are cached at the end of the build 33 | cache: 34 | directories: 35 | - $HOME/.ivy2/cache 36 | - $HOME/.sbt/boot/ 37 | - $HOME/.gitbook 38 | - node_modules/ 39 | 40 | env: 41 | global: 42 | secure: "DTM8LplUqjcBsRdvKv171K7I8V0kdD0Qsf/nIvuDsWfXbES4l1yqORDC25iHaZsld7lFfpyQ8bAWMUF8tWmwKyOc6h4IDqVVv8L3+U3aACf3NCe+XEdEP1uG/v2CqcaCUI9d6wD6e1VwGE7ebrsyMZkPoft2UGqhSpXHm29S9pg8LYkLcPcWzpBavaNT2iwHFmIhxuDbIqQZOhBtm6HPh7bN6jolwu+wQIvLM4Zlh1i+wgif0gVwhSU+mIDOBJKblBu+w67Tjie8d5CLmeZclOi5LrmphDgxqmouRvjWfm7HlLNJqYn19ocyWWd4n6mNL2ho9Vl4O+Z5kH3QOmF1NY/kG+YuLWRZ6h3QEXSfBuTtGGYs0KtRC5Frt0K3hJhcv9b+VTSjIDyQCv8skOTjPNUUMJcqxW+K6ySbmJNMl1crdmW0u2b9ngdzIl3o48WHkEAPhF5PkpFGZH5+8FjdWIz356SUaXJvFOh0d6Y+qsKd9ubcYyvUuCzdDUEjUylF4dZUoWVY75I+7xbUsvc23A8SfVy1KTiR3Ij1+40PcIw/zvApZw3DWcl47kqpv4N834LhhdG73oD08cgT5ta+/sifUIDQbU05elp634Yk9OOD0XlcMM3G6zdO4XQmYQ48rpXtZsAieunyVO0e5PeSUlsp08K5R8sCi//EereYRcs=" 43 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Bugs and feature requests should be reported in the [GitHub issue 4 | tracker](https://github.com/ypg-data/eigenflow/issues/new) and 5 | answer the following questions: 6 | 7 | - Motivation: Why should this be addressed? What is the purpose? 8 | - Input: What are the pre-conditions? 9 | - Output: What is the expected outcome after the issue has been addressed? 10 | - Test: How can the results listed in the "Output" be QA'ed? 11 | 12 | For code contributions, these are the suggested steps: 13 | 14 | - Identify the change you'd like to make, e.g. fix a bug or add a feature. 15 | Larger contributions should always begin with [first creating an 16 | issue](https://github.com/ypg-data/eigenflow/issues/new) to ensure 17 | that the change is properly scoped. 18 | - Fork the repository on GitHub. 19 | - Develop your change on a feature branch. 20 | - Write tests to validate your change works as expected. 21 | - Create a pull request. 22 | - Address any issues raised during the code review. 23 | - Once you get a "+1" on the pull request, the change can be merged. 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Eigenflow 2 | 3 | Eigenflow is an orchestration platform for building resilient and scalable data pipelines. 4 | 5 | Pipelines can be split into multiple process stages which are persisted, resumed and monitored automatically. 6 | 7 | [![Build Status](https://travis-ci.org/ypg-data/eigenflow.svg?branch=master)](https://travis-ci.org/ypg-data/eigenflow) 8 | [![Latest Version](https://api.bintray.com/packages/ypg-data/maven/eigenflow/images/download.svg)](https://bintray.com/ypg-data/maven/eigenflow/_latestVersion) 9 | 10 | Quick example: 11 | 12 | ```scala 13 | case object Download extends ProcessStage 14 | case object Transform extends ProcessStage 15 | case object Analyze extends ProcessStage 16 | case object SendReport extends ProcessStage 17 | 18 | val download = Download { 19 | downloadReport() // returns file path/url to downloaded report 20 | } retry (1.minute, 10) // in case of error retry every minute 10 times before failing 21 | 22 | val transform = Transform { reportFile => 23 | buildParquetFile(reportFile) // returns file path/url 24 | } 25 | 26 | val analyze = Analyze { parquetFile => 27 | callSparkToAnalyze(parquetFile) // returns new report file path/url 28 | } 29 | 30 | val sendReport = SendReport { newReportFile => 31 | sendReportToDashboard(newReportFile) 32 | } 33 | 34 | override def executionPlan = download ~> transform ~> analyze ~> sendReport 35 | ``` 36 | 37 | Once the stage methods (`downloadReport`, `buildParquetFile` etc in the example above) are defined the rest 38 | is done automatically: see [complete list of features](#main-features). 39 | 40 | ### What it is good for 41 | 42 | `Eigenflow` was created for managing periodic long-running ETL processes with automatic recovery of failures. 43 | When stages performance is important and there is a need to collect statistics and monitor processes. 44 | 45 | 46 | ### What it may not be good for 47 | 48 | `Eigenflow` is a platform somewhere between "simple cron jobs" and complex enterprise processes, 49 | where an ESB software would usually be used. 50 | Thus, it probably should not be considered for primitive jobs and very complex processes where SOA is involved. 51 | 52 | 53 | ## Main Features 54 | 55 | * Stages: DSL for building type safe data pipeline (Scala only). 56 | * Time Management: configurable strategy for catching up when "run cycles" are missing. 57 | * Recovery: if a process failed the platform starts replaying the failed stage, by default. 58 | * Error Handling: configurable recovery strategies per stage. 59 | * Metrics: Each `process run`, `stage switch or failure` and `custom messages` are published as events to a messaging system. 60 | Kafka is supported by default, but a custom messaging system adapter can be integrated. 61 | * Monitoring: provided as a module, which uses `grafana` and `influxDB` to store and display statistics. 62 | * Notifications: provided as a module, supports `email` and `slack` notifications. 63 | 64 | Custom monitoring and notification systems can be developed. 65 | Messages are pushed to a message queue (Kafka is supported out of the box) and can be consumed by a message queue consumer. 66 | 67 | Note: there is no connectors to 3rd party systems out of the box. 68 | 69 | 70 | ## System Requirements 71 | 72 | ### Runtime 73 | 74 | * JVM 8 75 | 76 | ### Development 77 | 78 | * Scala 2.11.0 or higher 79 | * Sbt 0.13.7 80 | * DevOps scripts are currently tested on Mac OS 10.10 only 81 | 82 | -------------------------------------------------------------------------------- /SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Summary 2 | 3 | - [Getting Started](doc/getting-started.md) 4 | - [Process Stages](doc/process-stages.md) 5 | - [Time Management](doc/time-management.md) 6 | - [Recovery](doc/recovery.md) 7 | - [Error Handling](doc/error-handling.md) 8 | - [Messaging](doc/messaging.md) 9 | - [DevOps](devops/README.md) 10 | - [Contributing](CONTRIBUTING.md) 11 | -------------------------------------------------------------------------------- /book.json: -------------------------------------------------------------------------------- 1 | { 2 | "gitbook": "2.x.x", 3 | "links": { 4 | "sidebar": { 5 | "Project": "https://github.com/ypg-data/eigenflow", 6 | "Issues": "https://github.com/ypg-data/eigenflow/issues" 7 | } 8 | }, 9 | "plugins": [ 10 | "toc", 11 | "mermaid-2" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | organization in ThisBuild := "com.mediative" 2 | name in ThisBuild := "eigenflow" 3 | scalaVersion in ThisBuild := "2.11.11" 4 | 5 | Jvm.`1.8`.required 6 | 7 | // Dependencies 8 | resolvers += Resolver.bintrayRepo("krasserm", "maven") 9 | 10 | val akkaVersion = "2.4.19" 11 | val kafkaVersion = "0.9.0.0" 12 | 13 | libraryDependencies ++= Seq( 14 | // akka 15 | "com.typesafe.akka" %% "akka-actor" % akkaVersion, 16 | "com.typesafe.akka" %% "akka-persistence" % akkaVersion, 17 | "com.github.krasserm" %% "akka-persistence-cassandra" % "0.6", 18 | 19 | // kafka 20 | "org.apache.kafka" %% "kafka" % kafkaVersion exclude("log4j", "log4j") exclude("org.slf4j","slf4j-log4j12"), 21 | "org.apache.kafka" % "kafka-clients" % kafkaVersion, 22 | 23 | // json 24 | "com.lihaoyi" %% "upickle" % "0.3.6", 25 | 26 | // test libraries 27 | "org.scalatest" %% "scalatest" % "2.2.6" % "test", 28 | "org.scalacheck" %% "scalacheck" % "1.12.5" % "test", 29 | "com.typesafe.akka" %% "akka-testkit" % akkaVersion % "test", 30 | "com.typesafe.akka" %% "akka-slf4j" % akkaVersion % "test", 31 | "ch.qos.logback" % "logback-classic" % "1.1.3" % "test" 32 | ) 33 | 34 | enablePlugins(MediativeReleasePlugin, MediativeBintrayPlugin) -------------------------------------------------------------------------------- /devops/README.md: -------------------------------------------------------------------------------- 1 | # DevOps 2 | 3 | For a quick start on Mac OS use the `devops/mac/eigenflow` script: 4 | 5 | Check docker installation and setup port forwarding 6 | 7 | $ ./eigenflow setup 8 | 9 | Start containers 10 | 11 | $ ./eigenflow start 12 | 13 | Note: The state will be reset on every start (empty database and message queues). 14 | 15 | 16 | Check the docker container status 17 | 18 | $ ./eigenflow state 19 | 20 | 21 | Print `Eigenflow` configuration suggestion 22 | 23 | $ ./eigenflow config 24 | 25 | Stop containers 26 | 27 | $ ./eigenflow stop 28 | -------------------------------------------------------------------------------- /devops/deploy-site.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Deploy doc site manually or from Travis to branch on GitHub. 4 | # 5 | # Travis requires a GitHub token in order to push changes to `gh-pages`. 6 | # This token can be generated from https://github.com/settings/tokens by 7 | # clicking [Generate new token](https://github.com/settings/tokens/new). 8 | # Copy the token to the clipboard. 9 | # 10 | # Next, install the Travis CLI and encrypt the token: 11 | # 12 | # $ gem install travis 13 | # $ travis encrypt GH_TOKEN= 14 | # Please add the following to your .travis.yml file: 15 | # 16 | # secure: "..." 17 | # 18 | # Copy the line starting with `secure:` and add it to `.travis.yml` 19 | # under `env` / `global`. 20 | # 21 | # env: 22 | # global: 23 | # secure: "..." 24 | # 25 | # See `.travis.yml` for how to configure Travis to call this script as 26 | # part of the `deploy` build stage. 27 | set -eu -o pipefail 28 | 29 | # The output directory for the generated docs. 30 | DOC_DIR="$(pwd)/_book" 31 | DOC_BRANCH=master 32 | 33 | if [ -z "${GH_TOKEN:-}" ]; then 34 | REPO_SLUG="$(git config remote.origin.url | sed -n 's#.*[:/]\(.*/.*\).git#\1#p')" 35 | REPO_URL_PREFIX="git@github.com:" 36 | else 37 | REPO_SLUG="$TRAVIS_REPO_SLUG" 38 | REPO_URL_PREFIX="https://$GH_TOKEN@github.com/" 39 | fi 40 | 41 | COMMIT="${TRAVIS_COMMIT:-$(git rev-parse HEAD)}" 42 | BUILD_ID="${TRAVIS_BUILD_NUMBER:-$(git symbolic-ref --short HEAD)}" 43 | BUILD_INFO="$REPO_SLUG@$COMMIT ($BUILD_ID)" 44 | 45 | REPO_ORG="${REPO_SLUG%/*}" 46 | REPO_NAME="${REPO_SLUG#*/}" 47 | REPO_CACHE=".git/$REPO_ORG.github.io" 48 | REPO_URL="$REPO_URL_PREFIX$REPO_ORG/$REPO_ORG.github.io.git" 49 | 50 | git init "$REPO_CACHE" 51 | 52 | cd "$REPO_CACHE" 53 | 54 | git config user.name "${USER}" 55 | git config user.email "${USER}@${COMMIT}" 56 | 57 | git fetch --depth=1 "$REPO_URL" 58 | git reset --hard FETCH_HEAD 59 | 60 | rm -rf "./$REPO_NAME" 61 | cp -a "$DOC_DIR" "./$REPO_NAME" 62 | git add "$REPO_NAME" 63 | 64 | if ! git diff --cached --exit-code --quiet; then 65 | git commit -m "Update $REPO_NAME docs from $BUILD_INFO" 66 | git push --force --quiet "$REPO_URL" "HEAD:$DOC_BRANCH" > /dev/null 2>&1 67 | fi 68 | -------------------------------------------------------------------------------- /devops/mac/eigenflow: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | vm_name="default" 4 | cql_port=9042 5 | internode_port=7000 6 | declare -a cassandra_ports=("$cql_port" "$internode_port") 7 | 8 | zookeeper_port=2181 9 | kafka_port=9092 10 | declare -a kafka_ports=("$zookeeper_port" "$kafka_port") 11 | kafka_docker="kafka" 12 | 13 | cassandra_image_name="cassandra" 14 | cassandra_docker="cassandra:2.2.4" 15 | kafka_image_name="kafka" 16 | kafka_docker="spotify/kafka" 17 | 18 | cqlsh_docker="cqlsh:2.1.11" 19 | 20 | function error { 21 | echo $1 22 | exit 1 23 | } 24 | 25 | function set_docker { 26 | docker-machine ls | grep -q $vm_name 27 | if [ $? -eq 1 ]; then 28 | docker-machine create --driver virtualbox $vm_name 29 | fi 30 | 31 | vm_status=`docker-machine status $vm_name` 32 | 33 | if [[ "$vm_status" != "Running" ]]; then 34 | VBoxManage startvm $vm_name 35 | error "Wait until the VM starts and re-run the command" 36 | fi 37 | 38 | eval $(docker-machine env $vm_name) 39 | } 40 | 41 | function check_nat_rule { 42 | rule_name=$1 43 | port=$2 44 | 45 | VBoxManage showvminfo $vm_name | grep -q "host port = $port" 46 | if [ $? -eq 1 ]; then 47 | echo "setting up nat rule $rule_name" 48 | VBoxManage controlvm $vm_name natpf1 "$rule_name,tcp,,$port,,$port" 49 | fi 50 | } 51 | 52 | function setup_ports { 53 | declare -a ports=("${!1}") 54 | for port in "${ports[@]}" 55 | do 56 | check_nat_rule "tcp-$port" $port 57 | done 58 | } 59 | 60 | 61 | function run_kafka { 62 | docker ps -a -q --filter="name=$kafka_image_name" | xargs docker rm -f 63 | docker run --name $kafka_image_name -d -p $zookeeper_port:$zookeeper_port -p $kafka_port:$kafka_port --env ADVERTISED_HOST=$1 --env ADVERTISED_PORT=$kafka_port $kafka_docker 64 | } 65 | 66 | function run_cassandra { 67 | ip=$1 68 | docker ps -a -q --filter="name=$cassandra_image_name" | xargs docker rm -f 69 | docker run --name $cassandra_image_name -d -e CASSANDRA_BROADCAST_ADDRESS=$ip -p $cql_port:$cql_port -p $internode_port:$internode_port $cassandra_docker 70 | } 71 | 72 | function start { 73 | set_docker 74 | 75 | ip=`docker-machine ip $vm_name` 76 | 77 | run_cassandra $ip 78 | run_kafka $ip 79 | 80 | echo "To see the containers states run '$0 state'" 81 | echo "To connect to cassandra run '$0 cql'" 82 | echo "To generate eigenflow application config run '$0 config'" 83 | } 84 | 85 | function setup { 86 | command VBoxManage > /dev/null 2>&1 || error "Please install VirtualBox" 87 | command docker > /dev/null 2>&1 || error "Please install docker for Mac OS" 88 | command docker-machine > /dev/null 2>&1 || error "Please install docker for Mac OS" 89 | 90 | set_docker 91 | 92 | setup_ports cassandra_ports[@] 93 | setup_ports kafka_ports[@] 94 | 95 | echo "Done! run'$0 start'" 96 | } 97 | 98 | function config { 99 | ip=`docker-machine ip $vm_name` 100 | 101 | cat << EOM 102 | akka.persistence.journal.plugin = "cassandra-journal" 103 | cassandra-journal.contact-points = ["$ip:$cql_port"] 104 | 105 | eigenflow.kafka.bootstrap.servers = "$ip:$kafka_port" 106 | EOM 107 | } 108 | 109 | function stop { 110 | set_docker 111 | docker ps -q --filter="name=$kafka_image_name" | xargs docker stop > /dev/null 112 | docker ps -q --filter="name=$cassandra_image_name" | xargs docker stop > /dev/null 113 | } 114 | 115 | function status { 116 | set_docker 117 | docker ps 118 | } 119 | 120 | function cql { 121 | set_docker 122 | ip=`docker-machine ip $vm_name` 123 | docker ps -a -q --filter="name=cqlsh" | xargs docker rm -f > /dev/null 124 | docker run --name cqlsh -it --link $cassandra_image_name:cassandra --rm cassandra sh -c "cqlsh $cassandra_image_name" 125 | } 126 | 127 | function state { 128 | set_docker 129 | docker ps 130 | } 131 | 132 | case $1 in 133 | setup) 134 | setup 135 | ;; 136 | start) 137 | start 138 | ;; 139 | stop) 140 | stop 141 | ;; 142 | config) 143 | config 144 | ;; 145 | cql) 146 | cql 147 | ;; 148 | state) 149 | state 150 | ;; 151 | docker) 152 | docker-machine env $vm_name 153 | ;; 154 | *) 155 | error "USAGE: $0 setup|start|stop|state|config|cql|docker" 156 | ;; 157 | esac -------------------------------------------------------------------------------- /doc/error-handling.md: -------------------------------------------------------------------------------- 1 | # Error Handling 2 | 3 | By default, if no strategy is defined, when an exception happens the process switches to the `failed` stage remembering 4 | the stage it was trying to execute and exits. 5 | 6 | The simplest possible strategy would be: no matter what happens retry _n_-times with given interval: 7 | 8 | ```scala 9 | SomeStage { 10 | ... 11 | } retry (1.minute, 10) 12 | ``` 13 | 14 | You can define multiple strategies depending on exception type: 15 | 16 | ```scala 17 | SomeStage { 18 | ... 19 | } onFailure { 20 | case _: ConnectionTimeoutException => Retry(3.minutes, 10) 21 | case _: IOException => Retry(10.minutes, 3) 22 | } globalRecoveryTimeout(45.minutes) 23 | ``` 24 | 25 | `globalRecoveryTimeout` helps to limit the deadline on all retries. 26 | In the example above the potential maximum time the stage will be retrying is 1 hour 27 | (30 minutes for ConnectionTimeout and 30 minutes for IOException), 28 | but the `globalRecoveryTimeout` will limit it to ~45 minutes. 29 | Note: if a stage started execution, it won't be interrupted even if the timeout reached! 30 | 31 | `globalRecoveryTimeout` is optional, no limit by default. 32 | Thus the time spent on retries will be limited only by a sum of all `Retry` settings + time spent on actual executions. 33 | 34 | **IMPORTANT**: `globalRecoveryTimeout` applies only when stage is actually retrying, it has no effect on normal stage execution! 35 | If execution takes 5 hours and it does not throw any exceptions nothing will interrupt it! 36 | 37 | The uncovered exceptions will fail the process (default behaviour when no recovery strategy defined). 38 | 39 | If system crashed during stage retry it will restore all counters and timeout points. 40 | 41 | If system "normally" fails due to timeout or "tried max number of attempts" then all counters and timeouts will be reset on restart. 42 | 43 | Example of strategy recovery in different scenarios: 44 | 45 | 1. If multiple strategies are defined and the exceptions happen in a random order the system will keep counting 46 | number of retries per exception. 47 | For the example (for the code above) if exceptions happen in the following order: 48 | `ConnectionTimeoutException - IOException - IOException - ConnectionTimeoutException` the retries counter for 49 | `ConnectionTimeout` will not be reset when `IOException` is thrown. 50 | 1. If a system reboot/failure happens during stage retries, the system will restore counters and retries timeout on start. 51 | Note: if `globalRecoveryTimeout` is defined and the system was re-started after some delay it may be stopped due to timeout reached. 52 | But the next restart will reset all counters. 53 | 1. If number of retries exhausted or timeout reached the process fails. The new restart will reset all counters and timeout. 54 | -------------------------------------------------------------------------------- /doc/getting-started.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | For setting up an environment on Mac OS follow the instructions for the 4 | [DevOps script](../devops/README.md). 5 | 6 | ## SBT 7 | 8 | Add a library dependency on Eigenflow in `build.sbt` 9 | 10 | ```scala 11 | libraryDependencies += "com.mediative" %% "eigenflow" % "x.x.x" 12 | ``` 13 | 14 | ## Create Process 15 | 16 | Define custom process stages: 17 | 18 | ```scala 19 | import com.mediative.eigenflow.domain.fsm.ProcessStage 20 | 21 | case object Extract extends ProcessStage 22 | case object Transform extends ProcessStage 23 | case object Load extends ProcessStage 24 | ``` 25 | 26 | all stages must extend `ProcessStage` trait. 27 | 28 | Create a process class: 29 | 30 | ```scala 31 | class SimpleProcess extends StagedProcess { 32 | } 33 | ``` 34 | 35 | `StagedProcess` implementations must implement the `executionPlan` method, which describes the pipeline stages flow. 36 | Describe it using the DSL: 37 | 38 | ```scala 39 | val extract = Extract { 40 | ... 41 | } 42 | 43 | val transform = Transform { resultFromExtract => 44 | ... 45 | } 46 | 47 | val load = Load { resultFromTransform => 48 | ... 49 | } 50 | 51 | override def executionPlan: ExecutionPlan[_, _] = extract ~> transform ~> load 52 | ``` 53 | 54 | See [the Process Stages section](process-stages.md) for important details. 55 | 56 | ## Main Class 57 | 58 | ```scala 59 | object SimpleProcessApp extends App with EigenflowBootstrap { 60 | override def process: StagedProcess = new SimpleProcess 61 | } 62 | ``` 63 | 64 | ## Basic Configuration 65 | 66 | Create `resources/application.conf` file with the content: 67 | 68 | ```json 69 | process { 70 | id = "simpleProcess" 71 | } 72 | ``` 73 | 74 | Run it 75 | 76 | ``` 77 | sbt run 78 | ``` 79 | 80 | 81 | Note: by default it uses Akka inmem storage and PrintMessagingSystem (which simply prints messages to logs), what means it won't restore after a failure. 82 | 83 | To configure storage and messaging system use `application.conf` 84 | 85 | Example of storage configuration for a local cassandra installation 86 | 87 | ```json 88 | akka { 89 | persistence { 90 | journal { 91 | plugin = "cassandra-journal" 92 | } 93 | } 94 | } 95 | 96 | cassandra-journal { 97 | contact-points = ["localhost:9042"] 98 | } 99 | ``` 100 | 101 | Example of messaging system configuration for a local Kafka installation 102 | 103 | ```json 104 | eigenflow { 105 | messaging = "com.mediative.eigenflow.publisher.kafka.KafkaConfiguration" 106 | 107 | kafka { 108 | bootstrap.servers = "localhost:9092" 109 | } 110 | } 111 | ``` 112 | -------------------------------------------------------------------------------- /doc/messaging.md: -------------------------------------------------------------------------------- 1 | # Messaging 2 | 3 | 4 | 5 | ## Standard Messages 6 | 7 | `Eigenflow` publishes messages for each run start/failure/complete and stage switches. 8 | 9 | (Subject to change) Currently each process has it's own set of topics: 10 | 11 | - `process_id` for messages on process level: run start/failure/complete events 12 | - `process_id-stages` for messages on a process run level: stage switches 13 | - `process_id-` for metrics messages, see [Custom Messages](#custom-messages) 14 | 15 | Published messages are json serialized instances of classes defined in [messages package](https://github.com/ypg-data/eigenflow/tree/master/src/main/scala/com.mediative.eigenflow/domain/messages). 16 | 17 | ## Custom Messages 18 | 19 | There are 2 types of custom messages currently supported: 20 | 21 | ### Metrics messages 22 | 23 | Metrics messages must be added to each stage and provide `StageResult => Map[String, Double]` function to generate metrics. 24 | For example: 25 | 26 | ```scala 27 | val stage = Stage { 28 | Future { ... } 29 | } publishMetrics(_ => Map("total" -> 1.0)) 30 | ``` 31 | 32 | In this case a message with process, stage and metrics information will be published to `process_id-statistics` topic. 33 | 34 | ### Direct messaging 35 | 36 | To get access to messaging system define implicit of `MessagingSystem` in process class constructor, for example: 37 | 38 | ```scala 39 | class MyProcess(implicit ms: MessagingSystem) extends StagedProcess 40 | ``` 41 | 42 | now a message can be published using `publish` function: 43 | 44 | ``` 45 | ms.publish(topicName, message) 46 | ``` 47 | 48 | The `message` parameter can be any string. 49 | 50 | Suggestion: [GenericMessage](https://github.com/ypg-data/eigenflow/blob/master/src/main/scala/com.mediative.eigenflow/domain/messages/GenericMessage.scala) 51 | can be used to add basic process information fields (not data!) along with the custom message, but it's optional. 52 | 53 | ## Configuration 54 | 55 | By default `Eigenflow` uses `PrintMessagingSystem` which simply logs messages using the standard logger. 56 | 57 | To enable kafka the following settings must be set in `application.conf` 58 | 59 | ``` 60 | eigenflow { 61 | messaging = "com.mediative.eigenflow.publisher.kafka.KafkaMessagingSystem" 62 | 63 | kafka { 64 | bootstrap.servers = "..." 65 | topic.prefix = "..." // Optional. The Default value is "eigenflow". 66 | } 67 | } 68 | ``` 69 | -------------------------------------------------------------------------------- /doc/process-stages.md: -------------------------------------------------------------------------------- 1 | # Process Stages 2 | 3 | Eigenflow requires at least one stage to run the process. 4 | To define process stages create case objects which extend `ProcessStage` trait. For example: 5 | 6 | ```scala 7 | case object StageName extends ProcessStage 8 | ``` 9 | 10 | The stages are mainly used as checkpoints during the process. 11 | `Eigenflow` persists each stage completion and the result of that stage. 12 | The result of stage execution must be returned in a `future`, to define stage execution logic use DSL: 13 | 14 | ```scala 15 | StageName { 16 | process() 17 | } 18 | ``` 19 | 20 | the `process` function in this example must return a `Future[A]`, then the next stage will receive A as argument. 21 | The `A` can be anything but serialization/deserialization implicit functions must be available: 22 | 23 | ```scala 24 | A => String 25 | String => A 26 | ``` 27 | 28 | `Eigenflow` provides the `PrimitiveImplicits` object which can be imported to enable some primitives serialization. 29 | 30 | If you need access to process context, for example to calculate for which date to fetch the report you can use `withContext` method. 31 | Here is an example of downloading and archiving a report. 32 | 33 | ```scala 34 | case object Initial extends ProcessStage 35 | case object Download extends ProcessStage 36 | case object Archive extends ProcessStage 37 | 38 | val init = Initial withContext { context: ProcessContext => 39 | Future { 40 | context.processingDate.minusDays(1) // means always download report from yesterday (relating to the current processingDate) 41 | } 42 | } 43 | 44 | val download = Download { reportDate => 45 | downloadReportForDate(reportDate) // returns Future[String] which contains file name 46 | } 47 | 48 | val archive = Archive { fileName => 49 | archive(fileName) 50 | } 51 | 52 | override def executionPlan = init ~> download ~> archive 53 | ``` 54 | 55 | **IMPORTANT** 56 | - Design stages like a reboot may happen between stages execution, thus NEVER share a state 57 | between stages. Only, the standard way, when the result returned from one stage and passed to another stage is safe, 58 | this result must contain all information shared between stages. 59 | - Stages must run sequentially, thus the `Future` returned from stage must be the final 60 | one. If other futures are created during stage execution you should combine them all using `Future.sequence`. 61 | - It's NOT recommended to pass big data between stages, store data in a storage (file, db, hadoop etc.) 62 | and pass the information how to find it. 63 | -------------------------------------------------------------------------------- /doc/recovery.md: -------------------------------------------------------------------------------- 1 | # Recovery 2 | 3 | `Eigenflow` remembers the last stage and the processing date. 4 | Thus if a process fails or crashes, it will re-run the failed stage automatically when restarted. 5 | When processing is complete for a date the `nextProcessingDate` function will be called to define if it should "catch-up". 6 | If the `nextProcessingDate` returns a date in the past the process continues to run with the new date until the `nextProcessingDate` 7 | returns a date in future. 8 | 9 | Think of it as a time line with repeating stages and the system always tries to execute all stages up to now, 10 | if a stage cannot be complete, it stuck processing in a `stage-time` point. 11 | 12 | To control the time function see: [Time Management](time-management.md) 13 | -------------------------------------------------------------------------------- /doc/time-management.md: -------------------------------------------------------------------------------- 1 | # Time Management 2 | 3 | If a process must run once an hour, day, week etc. you can have an additional control which allows automatically 4 | 5 | * Catching up, for example if a process should run daily but the last successful run was 3 days ago it will automatically 6 | run for every missing day until now. 7 | * Protects from double run, if a process should run daily and it **successfully** finished execution today it won't run for today 8 | again even if it was executed again, unless it was explicitly asked to do so. 9 | Thus you shouldn't worry about occasional loading of the same data twice. 10 | Note: to have a guaranteed double run protection, the environment must satisfy 2 conditions: 11 | 1. Persistence layer must not be eventual consistent. We use cassandra configured for consistency. 12 | 1. No simultaneous runs of the same process. We use mesos with chronos to schedule processes in cluster. 13 | 14 | Override `nextProcessingDate(lastCompleted: Date): Date` method to define next run date should be, based on the last completed date. 15 | 16 | TODO: examples 17 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.15 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | resolvers += Resolver.bintrayIvyRepo("mediative", "sbt-plugins") 2 | 3 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.1") 4 | addSbtPlugin("com.mediative.sbt" % "sbt-mediative-core" % "0.5.6") 5 | addSbtPlugin("com.mediative.sbt" % "sbt-mediative-oss" % "0.5.6") 6 | -------------------------------------------------------------------------------- /src/it/resources/application.conf: -------------------------------------------------------------------------------- 1 | process { 2 | id = "test" 3 | } 4 | 5 | akka { 6 | loggers = ["akka.event.slf4j.Slf4jLogger"] 7 | } -------------------------------------------------------------------------------- /src/it/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | [%level] [%d{ISO8601}] %m%n 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /src/it/scala/com.mediative.eigenflow.test.it/process/ProcessFSMTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.test 18 | package it.process 19 | 20 | import java.util.Date 21 | 22 | import com.mediative.eigenflow.domain.fsm._ 23 | import com.mediative.eigenflow.test.it.utils.wrappers.TracedProcessFSM 24 | import TracedProcessFSM.Start 25 | import com.mediative.eigenflow.test.it.utils.EigenflowIntegrationTest 26 | 27 | import scala.concurrent.duration._ 28 | 29 | class ProcessFSMTest extends EigenflowIntegrationTest { 30 | "ProcessFSM" - { 31 | "for Stage1 ~> Stage2" - { 32 | "expect transition: Initial -> Stage1 -> Stage2 -> Complete" in { 33 | system.actorOf( 34 | TracedProcessFSM.props(`Stage1 ~> Stage2`, new Date, "test") 35 | ) ! Start 36 | 37 | val result = expectMsgType[Seq[ProcessStage]](5.seconds) 38 | 39 | assert(result == Seq(Initial, Stage1, Stage2, Complete)) 40 | } 41 | } 42 | 43 | "for Stage1(retry) ~> Stage2" - { 44 | "expect transition: Initial -> Stage1 -> Retry -> Stage1 -> Stage2 -> Complete" in { 45 | system.actorOf( 46 | TracedProcessFSM.props(`Stage1(retry) ~> Stage2`, new Date, "test") 47 | ) ! Start 48 | 49 | val result = expectMsgType[Seq[ProcessStage]](5.seconds) 50 | 51 | assert(result == Seq(Initial, Stage1, Retrying, Stage1, Stage2, Complete)) 52 | } 53 | } 54 | 55 | "for Stage1(Complete) ~> Stage2" - { 56 | "expect transition: Initial -> Stage1 -> Complete" in { 57 | system.actorOf( 58 | TracedProcessFSM.props(`Stage1(Complete) ~> Stage2`, new Date, "test") 59 | ) ! Start 60 | 61 | val result = expectMsgType[Seq[ProcessStage]](5.seconds) 62 | 63 | assert(result == Seq(Initial, Stage1, Complete)) 64 | } 65 | } 66 | 67 | "for Stage1 ~> Stage2(FailOnce) ~> Stage3" - { 68 | "expect transition: Initial -> Stage1 -> Stage2 -> Failed" in { 69 | system.actorOf( 70 | TracedProcessFSM.props(`Stage1 ~> Stage2(FailOnce) ~> Stage3`, new Date, "test") 71 | ) ! Start 72 | 73 | val result = expectMsgType[Seq[ProcessStage]](5.seconds) 74 | 75 | assert(result == Seq(Initial, Stage1, Stage2, Failed)) 76 | } 77 | 78 | "expect restart from the failed stage" in { 79 | val date = new Date 80 | system.actorOf( 81 | TracedProcessFSM.props(`Stage1 ~> Stage2(FailOnce) ~> Stage3`, date, "test") 82 | ) ! Start 83 | 84 | val result = expectMsgType[Seq[ProcessStage]](5.seconds) 85 | 86 | assert(result == Seq(Initial, Stage1, Stage2, Failed)) 87 | 88 | system.actorOf( 89 | TracedProcessFSM.props(`Stage1 ~> Stage2(FailOnce) ~> Stage3`, date, "test") 90 | ) ! Start 91 | 92 | val result2 = expectMsgType[Seq[ProcessStage]](5.seconds) 93 | 94 | assert(result2 == Seq(Failed, Stage2, Complete)) 95 | } 96 | 97 | "start over again if forced" in { 98 | val date = new Date 99 | system.actorOf( 100 | TracedProcessFSM.props(`Stage1 ~> Stage2(FailOnce) ~> Stage3`, date, "test") 101 | ) ! Start 102 | 103 | val result = expectMsgType[Seq[ProcessStage]](5.seconds) 104 | 105 | assert(result == Seq(Initial, Stage1, Stage2, Failed)) 106 | 107 | system.actorOf( 108 | TracedProcessFSM.props(`Stage1 ~> Stage2(FailOnce) ~> Stage3`, date, "test", reset = true) 109 | ) ! Start 110 | 111 | val result2 = expectMsgType[Seq[ProcessStage]](5.seconds) 112 | 113 | assert(result2 == Seq(Initial, Stage1, Stage2, Failed)) 114 | } 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/it/scala/com.mediative.eigenflow.test.it/process/ProcessManagerTest.scala: -------------------------------------------------------------------------------- 1 | package com.mediative.eigenflow.test.it.process 2 | 3 | import java.util.{ UUID, Date } 4 | 5 | import com.mediative.eigenflow.process.ProcessManager.{ Continue, ProcessingDateState } 6 | import com.mediative.eigenflow.test.it.utils.wrappers.TracedProcessManager 7 | import com.mediative.eigenflow.test.it.utils.models.{ FailingProcess, OneStageProcess } 8 | import com.mediative.eigenflow.test.it.utils.EigenflowIntegrationTest 9 | import org.scalacheck.Gen 10 | 11 | import scala.concurrent.duration._ 12 | 13 | class ProcessManagerTest extends EigenflowIntegrationTest { 14 | "ProcessManager" - { 15 | "when at least one or more run cycles missed" - { 16 | "must run process as many times as many runs are missing" in { 17 | 18 | // should be long enough so that all runs execution time will not exceed it, 19 | // otherwise it will execute one more cycle 20 | val intervalInMs = 10000L 21 | 22 | // add time interval to the previous date. 23 | def processingDate(intervalInMs: Long): Date => Date = date => new Date(date.getTime + intervalInMs) 24 | 25 | // generate tests with 1-3 run cycles. 26 | forAll(Gen.choose(1, 3), minSuccessful(4)) { runs => 27 | val persistenceId = Some(UUID.randomUUID().toString) 28 | 29 | val dateIterator = processingDate(intervalInMs) 30 | val now = System.currentTimeMillis() 31 | 32 | // start with the time sufficient to run exactly the given number of cycles 33 | val startDate = new Date(now - runs * intervalInMs) 34 | val process = OneStageProcess.create(dateIterator) 35 | 36 | val processManager = system.actorOf(TracedProcessManager.props( 37 | process = process, 38 | receiver = self, 39 | date = Some(startDate), 40 | id = persistenceId)) 41 | 42 | processManager ! Continue 43 | 44 | val dates = Iterator.iterate(startDate)(dateIterator). 45 | takeWhile(date => date.before(new Date(now)) || date.equals(new Date(now))). 46 | flatMap(date => Seq(ProcessingDateState(date, false), ProcessingDateState(date, true))).toSeq 47 | 48 | expectMsgAllOf[ProcessingDateState](5.seconds, dates: _*) 49 | 50 | expectNoMsg(2.seconds) 51 | } 52 | } 53 | } 54 | 55 | "when run is complete" - { 56 | "must not run until the next processing date" in { 57 | val persistenceId = Some(UUID.randomUUID().toString) 58 | val farInFuture = 1000000000L 59 | 60 | val process = OneStageProcess.create(_ => new Date(System.currentTimeMillis() + farInFuture)) 61 | val processManager = system.actorOf(TracedProcessManager.props( 62 | process = process, 63 | receiver = self, 64 | id = persistenceId)) 65 | 66 | processManager ! Continue 67 | // expect exactly 2 messages: first for run registration second for run complete 68 | expectMsgType[ProcessingDateState] 69 | expectMsgType[ProcessingDateState] 70 | expectNoMsg 71 | 72 | // simulate restart by creating ProcessManager with the same persistenceId 73 | val processManager2 = system.actorOf(TracedProcessManager.props( 74 | process = process, 75 | receiver = self, 76 | id = persistenceId)) 77 | 78 | processManager2 ! Continue 79 | // since the process was completed in previous run and the next processing date should be in the future 80 | // no runs should be executed, thus no messages received. 81 | expectNoMsg 82 | } 83 | 84 | "must re-run if start date is set" in { 85 | val persistenceId = Some(UUID.randomUUID().toString) 86 | val farInFuture = 1000000000L 87 | 88 | val process = OneStageProcess.create(_ => new Date(System.currentTimeMillis() + farInFuture)) 89 | 90 | val startDate = new Date // fix the date to be able to restart from the same point 91 | 92 | val processManager = system.actorOf(TracedProcessManager.props( 93 | process = process, 94 | receiver = self, 95 | date = Some(startDate), 96 | id = persistenceId)) 97 | 98 | processManager ! Continue 99 | // expect exactly 2 messages: first for run registration second for run complete 100 | expectMsgType[ProcessingDateState] 101 | expectMsgType[ProcessingDateState] 102 | expectNoMsg 103 | 104 | val processManager2 = system.actorOf(TracedProcessManager.props( 105 | process = process, 106 | receiver = self, 107 | date = Some(startDate), 108 | id = persistenceId)) 109 | 110 | processManager2 ! Continue 111 | // same messages again, for re-run was forced by startDate 112 | expectMsgAllOf(ProcessingDateState(startDate, false), ProcessingDateState(startDate, true)) 113 | expectNoMsg 114 | 115 | } 116 | } 117 | 118 | "when run fails" - { 119 | "must not persist complete status" in { 120 | val farInFuture = 1000000000L 121 | val process = FailingProcess.create(_ => new Date(System.currentTimeMillis() + farInFuture)) 122 | 123 | val processManager = system.actorOf(TracedProcessManager.props(process = process, receiver = self)) 124 | processManager ! Continue 125 | val response = expectMsgType[ProcessingDateState] 126 | expectNoMsg() 127 | 128 | assert(!response.complete) 129 | } 130 | 131 | "must restart from the failed run" in { 132 | val persistenceId = Some(UUID.randomUUID().toString) 133 | val farInFuture = 1000000000L 134 | val process = FailingProcess.create(_ => new Date(System.currentTimeMillis() + farInFuture)) 135 | 136 | val processManager = system.actorOf(TracedProcessManager.props(process = process, receiver = self, id = persistenceId)) 137 | processManager ! Continue 138 | val response = expectMsgType[ProcessingDateState] 139 | expectNoMsg() 140 | 141 | assert(!response.complete) 142 | 143 | val processManager2 = system.actorOf(TracedProcessManager.props(process = process, receiver = self, id = persistenceId)) 144 | processManager2 ! Continue 145 | val response2 = expectMsgType[ProcessingDateState] 146 | expectNoMsg() 147 | 148 | assert(response.date.equals(response2.date)) 149 | } 150 | } 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/it/scala/com.mediative.eigenflow.test.it/utils/EigenflowIntegrationTest.scala: -------------------------------------------------------------------------------- 1 | package com.mediative.eigenflow.test.it.utils 2 | 3 | import akka.actor.ActorSystem 4 | import akka.testkit.{ ImplicitSender, TestKit } 5 | import com.mediative.eigenflow.dsl.EigenflowDSL 6 | import org.scalatest.{ BeforeAndAfterAll, Matchers, FreeSpecLike } 7 | import org.scalatest.concurrent.ScalaFutures 8 | import org.scalatest.prop.{ Checkers, GeneratorDrivenPropertyChecks } 9 | 10 | class EigenflowIntegrationTest(_system: ActorSystem) 11 | extends TestKit(_system) with ImplicitSender 12 | with FreeSpecLike with ScalaFutures with GeneratorDrivenPropertyChecks with Matchers with BeforeAndAfterAll 13 | with Checkers 14 | with EigenflowDSL { 15 | 16 | def this() = this(ActorSystem("EigenflowTestActorSystem")) 17 | 18 | override def afterAll: Unit = { 19 | TestKit.shutdownActorSystem(system) 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/it/scala/com.mediative.eigenflow.test.it/utils/models/FailingProcess.scala: -------------------------------------------------------------------------------- 1 | package com.mediative.eigenflow.test.it.utils.models 2 | 3 | import java.util.Date 4 | 5 | import com.mediative.eigenflow.StagedProcess 6 | import com.mediative.eigenflow.domain.fsm.ExecutionPlan 7 | import com.mediative.eigenflow.test.Stage1 8 | 9 | import scala.concurrent.Future 10 | import scala.concurrent.ExecutionContext.Implicits.global 11 | 12 | object FailingProcess { 13 | def create(f: Date => Date): FailingProcess = new FailingProcess { 14 | override def nextProcessingDate(lastCompleted: Date): Date = f(lastCompleted) 15 | } 16 | } 17 | 18 | trait FailingProcess extends StagedProcess { 19 | val a = Stage1 { 20 | Future[String] { 21 | throw new RuntimeException 22 | } 23 | } 24 | 25 | override def executionPlan: ExecutionPlan[_, _] = a 26 | } 27 | -------------------------------------------------------------------------------- /src/it/scala/com.mediative.eigenflow.test.it/utils/models/OneStageProcess.scala: -------------------------------------------------------------------------------- 1 | package com.mediative.eigenflow.test.it.utils.models 2 | 3 | import java.util.Date 4 | 5 | import com.mediative.eigenflow.StagedProcess 6 | import com.mediative.eigenflow.domain.fsm.ExecutionPlan 7 | import com.mediative.eigenflow.test.Stage1 8 | 9 | import scala.concurrent.ExecutionContext.Implicits.global 10 | import scala.concurrent.Future 11 | 12 | object OneStageProcess { 13 | def create(f: Date => Date): StagedProcess = new OneStageProcess { 14 | override def nextProcessingDate(lastCompleted: Date): Date = f(lastCompleted) 15 | } 16 | } 17 | 18 | trait OneStageProcess extends StagedProcess { 19 | val a = Stage1 { 20 | Future { "" } 21 | } 22 | 23 | override def executionPlan: ExecutionPlan[_, _] = a 24 | } -------------------------------------------------------------------------------- /src/it/scala/com.mediative.eigenflow.test.it/utils/wrappers/TracedProcessFSM.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.test.it.utils.wrappers 18 | 19 | import java.util.Date 20 | 21 | import akka.actor.Props 22 | import akka.event.LoggingAdapter 23 | import com.mediative.eigenflow.StagedProcess 24 | import com.mediative.eigenflow.domain.ProcessContext 25 | import com.mediative.eigenflow.domain.fsm.{ ProcessEvent, ProcessStage } 26 | import com.mediative.eigenflow.process.ProcessFSM 27 | import com.mediative.eigenflow.process.ProcessFSM.Continue 28 | import com.mediative.eigenflow.publisher.MessagingSystem 29 | import com.typesafe.config.ConfigFactory 30 | 31 | object TracedProcessFSM { 32 | 33 | implicit def messagingSystem = new MessagingSystem(ConfigFactory.empty()) { 34 | override def publish(topic: String, message: String)(implicit log: LoggingAdapter): Unit = () // ignore 35 | } 36 | 37 | def props(process: StagedProcess, date: Date, processTypeId: String, reset: Boolean = false) = Props(new TracedProcessFSM(process, date, processTypeId, reset)) 38 | 39 | case object Start 40 | 41 | class TracedProcessFSM(process: StagedProcess, date: Date, processTypeId: String, reset: Boolean) extends ProcessFSM(process, date, processTypeId, reset) { 42 | private var originalSender = sender 43 | private var stagesRegistry: Seq[ProcessStage] = Seq.empty 44 | 45 | def testReceive: PartialFunction[Any, Unit] = { 46 | case Start => 47 | originalSender = sender() 48 | self ! Continue 49 | } 50 | 51 | override def applyEvent(domainEvent: ProcessEvent, processContext: ProcessContext): ProcessContext = { 52 | // register the last stage just before switching it, but only for the current run. 53 | // if this actor is restored form a previous run, the history recovery will not be registered, 54 | // it allows to reason about stages within one run context. 55 | // if a need for the full history arises then another registry should be created. 56 | if (!recoveryRunning) { 57 | stagesRegistry = stagesRegistry.:+(stateName) 58 | } 59 | super.applyEvent(domainEvent, processContext) 60 | } 61 | 62 | override def postStop(): Unit = { 63 | stagesRegistry = stagesRegistry.:+(stateName) // add the last state 64 | originalSender ! stagesRegistry 65 | super.postStop() 66 | } 67 | 68 | override def receive = testReceive orElse super.receive 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/it/scala/com.mediative.eigenflow.test.it/utils/wrappers/TracedProcessManager.scala: -------------------------------------------------------------------------------- 1 | package com.mediative.eigenflow.test.it.utils.wrappers 2 | 3 | import java.util.Date 4 | 5 | import akka.actor.{ ActorRef, Props } 6 | import akka.event.LoggingAdapter 7 | import com.mediative.eigenflow.StagedProcess 8 | import com.mediative.eigenflow.process.ProcessManager 9 | import com.mediative.eigenflow.process.ProcessManager.ProcessingDateState 10 | import com.mediative.eigenflow.publisher.MessagingSystem 11 | import com.typesafe.config.ConfigFactory 12 | 13 | object TracedProcessManager { 14 | private implicit def messagingSystem = new MessagingSystem(ConfigFactory.empty()) { 15 | override def publish(topic: String, message: String)(implicit log: LoggingAdapter): Unit = () // ignore 16 | } 17 | 18 | def props(process: StagedProcess, 19 | receiver: ActorRef, 20 | date: Option[Date] = None, 21 | id: Option[String] = None) = Props(new TracedProcessManager(process, date, receiver, id)) 22 | 23 | class TracedProcessManager(process: StagedProcess, 24 | date: Option[Date], 25 | receiver: ActorRef, 26 | id: Option[String] = None) extends ProcessManager(process, date, id.getOrElse("test")) { 27 | 28 | override def persistenceId: String = id.getOrElse(super.persistenceId) 29 | 30 | override def persist[A](event: A)(handler: (A) => Unit): Unit = { 31 | event match { 32 | case p @ ProcessingDateState(_, _) => 33 | receiver ! p 34 | } 35 | super.persist(event)(handler) 36 | } 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/resources/mesos.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | persistence { 3 | journal { 4 | plugin = "cassandra-journal" 5 | } 6 | 7 | snapshot-store { 8 | plugin = "cassandra-snapshot-store" 9 | } 10 | } 11 | 12 | log-dead-letters-during-shutdown = off 13 | log-dead-letters = 0 14 | } 15 | 16 | cassandra-journal { 17 | contact-points = [ 18 | "cassandra-node1.marathon.mesos:9042", 19 | "cassandra-node2.marathon.mesos:9042", 20 | "cassandra-node3.marathon.mesos:9042"] 21 | replication-factor = 3 22 | write-consistency = "QUORUM" 23 | read-consistency = "QUORUM" 24 | } 25 | 26 | cassandra-snapshot-store { 27 | contact-points = [ 28 | "cassandra-node1.marathon.mesos:9042", 29 | "cassandra-node2.marathon.mesos:9042", 30 | "cassandra-node3.marathon.mesos:9042"] 31 | replication-factor = 3 32 | write-consistency = "QUORUM" 33 | read-consistency = "QUORUM" 34 | } 35 | 36 | eigenflow { 37 | messaging = "com.mediative.eigenflow.publisher.kafka.KafkaConfiguration" 38 | 39 | kafka { 40 | bootstrap.servers = "broker-1.kafka.mesos:6000,broker-2.kafka.mesos:6000" 41 | topic.prefix = "eigenflow" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | persistence { 3 | journal { 4 | plugin = "akka.persistence.journal.inmem" 5 | refresh-interval = 3s 6 | } 7 | view { 8 | auto-update-interval = 10ms 9 | } 10 | snapshot-store { 11 | plugin = "akka.persistence.snapshot-store.local" 12 | } 13 | } 14 | 15 | log-dead-letters-during-shutdown = off 16 | log-dead-letters = 0 17 | } 18 | 19 | eigenflow { 20 | messaging = "com.mediative.eigenflow.publisher.PrintMessagingSystem" 21 | 22 | kafka { 23 | acks = "all" 24 | retries = 0 25 | batch.size = 16384 26 | linger.ms = 1 27 | buffer.memory = 33554432 28 | key.serializer = "org.apache.kafka.common.serialization.StringSerializer" 29 | value.serializer = "org.apache.kafka.common.serialization.StringSerializer" 30 | topic.prefix = "eigenflow" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/EigenflowBootstrap.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow 18 | 19 | import akka.actor.ActorSystem 20 | import com.mediative.eigenflow.environment.ConfigurationLoader 21 | import com.mediative.eigenflow.helpers.DateHelper._ 22 | import com.mediative.eigenflow.process.ProcessManager.Continue 23 | import com.mediative.eigenflow.process.ProcessSupervisor 24 | import com.mediative.eigenflow.publisher.MessagingSystem 25 | import com.typesafe.config.Config 26 | 27 | import scala.concurrent._ 28 | import scala.concurrent.duration._ 29 | import scala.util.{ Failure, Success, Try } 30 | 31 | /** 32 | * Creates the actor system and start processing. 33 | * Mix it in with App to run the process from the main program. 34 | */ 35 | trait EigenflowBootstrap { 36 | /** 37 | * Create the process to run. 38 | */ 39 | def process: StagedProcess 40 | 41 | private val config: Config = loadConfig() 42 | protected def loadConfig(): Config = ConfigurationLoader.config 43 | 44 | // bootstrap the system: initialize akka, message publisher ... 45 | implicit val messagingSystem = MessagingSystem.create(config) 46 | 47 | // load environment variables 48 | private val startDate = Option(System.getenv("start")).flatMap(parse) 49 | 50 | // initialize actor system as late as possible (fix for Issue #29) 51 | implicit val system = ActorSystem("DataFlow", config) 52 | 53 | private val processTypeId = config.getString("process.id") 54 | 55 | // create main actor and tell to proceed. 56 | system.actorOf(ProcessSupervisor.props(process, startDate, processTypeId, stopSystem(system))) ! Continue 57 | 58 | /** 59 | * System shutdown logic. 60 | */ 61 | private def stopSystem(system: ActorSystem)(code: Int): Unit = { 62 | Try(messagingSystem.stop()) 63 | 64 | val terminationProcess = system.terminate() 65 | 66 | import scala.concurrent.ExecutionContext.Implicits.global 67 | 68 | val timeoutFuture = Future { //in case system.terminate hangs 69 | Thread.sleep(10000) 70 | throw new TimeoutException("actorsystem.terminate timed out. Force shutdown.") 71 | } 72 | 73 | Future.firstCompletedOf(List(terminationProcess, timeoutFuture)).onComplete { 74 | case Success(_) => 75 | System.exit(code) 76 | case Failure(ex) => 77 | ex.printStackTrace() 78 | Runtime.getRuntime.halt(1) 79 | } 80 | 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/StagedProcess.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow 18 | 19 | import java.util.Date 20 | 21 | import com.mediative.eigenflow.domain.fsm.ExecutionPlan 22 | import com.mediative.eigenflow.dsl.EigenflowDSL 23 | 24 | /** 25 | * The main trait which allows to define the execution stages. 26 | * 27 | */ 28 | trait StagedProcess extends EigenflowDSL { 29 | def executionPlan: ExecutionPlan[_, _] 30 | 31 | def initialProcessingDate: Date = new Date() 32 | 33 | def nextProcessingDate(lastCompleted: Date): Date 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/domain/ProcessContext.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.domain 18 | 19 | import java.util.{ UUID, Date } 20 | 21 | import com.mediative.eigenflow.domain.fsm.{ Initial, ProcessStage } 22 | 23 | /** 24 | * A set of basic attributes which help to maintain stage transitions logic 25 | * and carry over data between stages within one process. 26 | * 27 | * @param timestamp Timestamp of the last stage transition (except Retrying stage!). 28 | * @param processingDate Current processing date. 29 | * 30 | * The processingDate is important for recovery process, and for ability to catch up. 31 | * 32 | * for example: 33 | * A process should run every day, but the last run was 3 days ago. 34 | * If the 'nextProcessingDate' is incrementing by 1 day, the system will automatically process 35 | * all reports day be day until now. 36 | * 37 | * Note: the dates used for processing are not necessarily the same as the processingDate. 38 | * For example: A report's date could be "processingDate - 1 day". 39 | * 40 | * @param processId Process identification. 41 | * @param startTime Time when the process started. 42 | * @param stage The current stage. 43 | * @param failure Last failure of the current stage, if any. 44 | * @param retryRegistry Retry registry keeps records of retry logic for the current stage. 45 | * @param message Serialized data from the previous stage. 46 | * 47 | */ 48 | case class ProcessContext(timestamp: Long = System.currentTimeMillis(), 49 | processingDate: Date, 50 | processId: String, 51 | startTime: Long, 52 | stage: ProcessStage, 53 | failure: Option[Throwable] = None, 54 | retryRegistry: Option[RetryRegistry] = None, 55 | message: String) 56 | 57 | object ProcessContext { 58 | def default(processingDate: Date) = ProcessContext( 59 | timestamp = System.currentTimeMillis(), 60 | processingDate = processingDate, 61 | processId = UUID.randomUUID().toString, 62 | startTime = System.currentTimeMillis(), 63 | stage = Initial, 64 | message = "" 65 | ) 66 | } 67 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/domain/RecoveryStrategy.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.domain 18 | 19 | import scala.concurrent.duration.FiniteDuration 20 | 21 | sealed trait RecoveryStrategy 22 | 23 | object RecoveryStrategy { 24 | case class Retry(interval: FiniteDuration, attempts: Int) extends RecoveryStrategy 25 | 26 | case object Fail extends RecoveryStrategy 27 | 28 | case object Complete extends RecoveryStrategy 29 | 30 | def default: PartialFunction[Throwable, RecoveryStrategy] = { 31 | case _ => Fail 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/domain/RetryRegistry.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.domain 18 | 19 | import com.mediative.eigenflow.domain.RecoveryStrategy.Retry 20 | 21 | /** 22 | * Registry for the stage retries. 23 | * 24 | * Keeps tracking of: 25 | * - when the registry was created (first exception happened) 26 | * - timeout indicating how long to try (in milliseconds) 27 | * - failures registry. 28 | */ 29 | case class RetryRegistry(timestamp: Long, timeout: Option[Long] = None, failuresRegistry: Seq[FailureRegistry]) 30 | 31 | case class FailureRegistry(failure: Throwable, attemptsMade: Int, maxAttempts: Int) 32 | 33 | /** 34 | * Helper functions for retry registry. 35 | */ 36 | object RetryRegistry { 37 | /** 38 | * Create retry registry with the given timeout. 39 | * 40 | * @param timeout Global timeout of retries. None - for no timeout. 41 | */ 42 | def create(timeout: Option[Long]): RetryRegistry = RetryRegistry( 43 | timestamp = System.currentTimeMillis(), 44 | timeout = timeout, 45 | failuresRegistry = Seq.empty 46 | ) 47 | 48 | /** 49 | * Add a failure to the given registry. 50 | * If the registry does not exists it will create a new one with the new failure. 51 | * 52 | * @param registryOption Existing registry. None, if this is the first failure. 53 | * @param failure Failure to register. 54 | * @param retry Retry strategy applied for the given failure. 55 | * @param timeout The global timeout, is used only when creating a new registry. 56 | */ 57 | def addFailure(registryOption: Option[RetryRegistry], 58 | failure: Throwable, 59 | retry: Retry, 60 | timeout: Option[Long]): RetryRegistry = { 61 | val registry = registryOption.getOrElse(create(timeout)) // get existing or create a new registry if not exist. 62 | val updatedFailuresRegistry = findFailure(Some(registry), failure).fold { 63 | // first time this failure happened, add it to the registry 64 | registry.failuresRegistry.:+(FailureRegistry(failure, 1, retry.attempts)) 65 | } { failureRegistryEntry => 66 | // this failure already happened, increase the number of attempts. 67 | registry.failuresRegistry. 68 | filterNot(_ == failureRegistryEntry). 69 | :+(failureRegistryEntry. 70 | copy(attemptsMade = failureRegistryEntry.attemptsMade + 1)) 71 | } 72 | // update the registry 73 | registry.copy(failuresRegistry = updatedFailuresRegistry) 74 | } 75 | 76 | /** 77 | * Find a record of the given failure. 78 | * 79 | * @param registry Registry to search in. 80 | * @param failure Failure to search for. 81 | * @return Failure entry or None if nothing found. 82 | */ 83 | def findFailure(registry: Option[RetryRegistry], failure: Throwable): Option[FailureRegistry] = { 84 | registry.flatMap(_.failuresRegistry.find(_.failure.getClass.equals(failure.getClass))) 85 | } 86 | 87 | /** 88 | * Check if there are attempts left for the given failure or the timeout is not reached. 89 | * 90 | */ 91 | def isValidForNextRetry(registryOption: Option[RetryRegistry], failure: Throwable): Boolean = { 92 | registryOption.fold(true) { registry => 93 | registry.timeout.fold { 94 | // no timeout applied, validate if the number of attempts is not reached 95 | !isNumberOfAttemptsReached(registryOption, failure) 96 | } { deadline => 97 | // check if deadline and number of attempts are not reached 98 | (registry.timestamp + deadline < System.currentTimeMillis()) && 99 | !isNumberOfAttemptsReached(registryOption, failure) 100 | } 101 | } 102 | } 103 | 104 | /** 105 | * Check if the number of attempts are not reached for the given failure. 106 | * 107 | */ 108 | def isNumberOfAttemptsReached(registryOption: Option[RetryRegistry], failure: Throwable): Boolean = { 109 | registryOption.fold(false) { registry => 110 | findFailure(Some(registry), failure).fold(false)(entry => entry.attemptsMade >= entry.maxAttempts) 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/domain/fsm/ExecutionPlan.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.domain.fsm 18 | 19 | import com.mediative.eigenflow.domain.{ ProcessContext, RecoveryStrategy } 20 | 21 | import scala.concurrent.Future 22 | 23 | /** 24 | * ExecutionPlan is a linked data structure which describes: 25 | * - stage transitions 26 | * - error handling strategy per stage 27 | * - custom messages for publishing 28 | * 29 | * @param stage The stage the plan describes. 30 | * @param f Stage business logic. 31 | * @param from Deserializer from String to A. 32 | * Where the string is a serialized result of the previous stage and A is expected input type for this stage. 33 | * @param to Serializer from B to String. Serializes the execution result of this stage to String. 34 | * @param previous Link to previous stage. None for the first stage. 35 | * @param recoveryStrategy A strategy to handle exceptions thrown during stage execution. 36 | * If no strategy defined for a particular exception the stage will fail. 37 | * @param recoveryTimeout Time limit for the stage retries. 38 | * When one retry strategy is defined it's fairly easy to calculate timeout: 39 | * ~ (interval + stage_execution_time_before_exception) * number_of_times 40 | * If multiple strategies are defined and different exceptions are thrown time to time 41 | * the total timeout will be a sum of all timeouts. To limit this value globally use 42 | * this timeout parameter, it will stop retrying even if not all attempts are made. 43 | * 44 | * If timeout reached the stage fails. 45 | * 46 | * Note: this timeout will NOT interrupt a normal run, it will only have effect on recovery. 47 | * @param publishMetricsMap A map which allows to calculate different values based on the stage execution result and 48 | * publish those values to the given topic. Typical use case is publishing some execution 49 | * statistical data, like number of records processed or size of file downloaded etc. 50 | * @tparam A Expected input data type. 51 | * @tparam B Stage execution result type. 52 | */ 53 | case class ExecutionPlan[A, B]( 54 | stage: ProcessStage, f: ProcessContext => A => Future[B], 55 | from: String => A, to: B => String, 56 | recoveryStrategy: Throwable => RecoveryStrategy = RecoveryStrategy.default, recoveryTimeout: Option[Long] = None, 57 | previous: Option[ExecutionPlan[_, A]] = None, 58 | publishMetricsMap: Option[B => Map[String, Double]] = None) 59 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/domain/fsm/ProcessEvent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.domain.fsm 18 | 19 | import com.mediative.eigenflow.domain.RecoveryStrategy.Retry 20 | 21 | /** 22 | * Events of this type are persisted and replayed by akka PersistentFSM to restore the latest state. 23 | * 24 | * They are used internally by eigenflow to control context changes, logging and events publishing on stage transitions. 25 | */ 26 | sealed trait ProcessEvent 27 | 28 | case class StageComplete(nextStage: ProcessStage, result: String) extends ProcessEvent 29 | 30 | case class StageFailed(failure: Throwable) extends ProcessEvent 31 | 32 | case class StageRetry(failure: Throwable, retry: Retry, timeout: Option[Long]) extends ProcessEvent 33 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/domain/fsm/ProcessStage.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.domain.fsm 18 | 19 | import akka.persistence.fsm.PersistentFSM.FSMState 20 | 21 | /** 22 | * A state trait for FSM, extend it for custom stages. 23 | */ 24 | trait ProcessStage extends FSMState { 25 | def identifier = toString 26 | } 27 | 28 | /** 29 | * Always the first stage of any process. 30 | */ 31 | case object Initial extends ProcessStage 32 | 33 | /** 34 | * Final stage of a successfully ended process. 35 | */ 36 | case object Complete extends ProcessStage 37 | 38 | /** 39 | * Indicates that a stage failed, for either unhandled exception or exhausted retries. 40 | */ 41 | case object Failed extends ProcessStage 42 | 43 | /** 44 | * Indicates that there is a stage failed and is waiting for a scheduled retry. 45 | */ 46 | case object Retrying extends ProcessStage 47 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/domain/messages/GenericMessage.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.domain.messages 18 | 19 | /** 20 | * A generic message to be sent on stage complete. 21 | * 22 | * @param timestamp Time when the message was created. 23 | * @param runId Identification of the running process for which the message was created. 24 | * @param stage Stage name for which the message was created. 25 | * @param message A generic message. 26 | */ 27 | case class GenericMessage(timestamp: Long, runId: String, stage: String, message: String) 28 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/domain/messages/MetricsMessage.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.domain.messages 18 | 19 | /** 20 | * A higher level GenericMessage which allows to define a map of values instead of plain string. 21 | * Serialization to json format is handled by platform. 22 | * 23 | * @see GenericMessage 24 | */ 25 | case class MetricsMessage( 26 | timestamp: Long, 27 | jobId: String, 28 | processId: String, 29 | stage: String, 30 | message: Map[String, Double]) 31 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/domain/messages/ProcessMessage.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.domain.messages 18 | 19 | /** 20 | * Message for process state changes. 21 | * 22 | * @param timestamp Time when the message was created. 23 | * @param jobId Job identifier 24 | * @param processId Process identification. 25 | * @param processingDate Processing date of the process. 26 | * @param state Process state. 27 | * @param duration How long did it take to switch to the given state. 28 | * Initial state's duration should be 0. 29 | * @param message Additional information relevant to the process state. 30 | * For example: the complete state message will be the next processing date, 31 | * or the failed state message will contain exception details. 32 | */ 33 | case class ProcessMessage( 34 | timestamp: Long, 35 | jobId: String, 36 | processId: String, 37 | processingDate: String, 38 | state: String, 39 | duration: Long, 40 | message: String) 41 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/domain/messages/StageMessage.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.domain.messages 18 | 19 | /** 20 | * Message for stage state changes. 21 | * 22 | * @param timestamp Time when the message was created. 23 | * @param processId Process identification. 24 | * @param stage Stage name. 25 | * @param state Stage state. 26 | * @param duration How long did it take to switch to the given state. 27 | * Initial state's duration should be 0. 28 | * @param message Data passed between stages. 29 | */ 30 | case class StageMessage( 31 | timestamp: Long, 32 | jobId: String, 33 | processId: String, 34 | stage: String, 35 | state: String, 36 | duration: Long, 37 | message: String) 38 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/domain/messages/StateRecord.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.domain.messages 18 | 19 | /** 20 | * Process and stage states registry. 21 | */ 22 | sealed trait StateRecord { 23 | def identifier = toString 24 | } 25 | 26 | case object Processing extends StateRecord 27 | 28 | case object Retrying extends StateRecord 29 | 30 | case object Failed extends StateRecord 31 | 32 | case object Complete extends StateRecord 33 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/dsl/EigenflowDSL.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.dsl 18 | 19 | import com.mediative.eigenflow.domain.RecoveryStrategy.Retry 20 | import com.mediative.eigenflow.domain.fsm.{ ExecutionPlan, ProcessStage } 21 | import com.mediative.eigenflow.domain.{ ProcessContext, RecoveryStrategy } 22 | 23 | import scala.concurrent.Future 24 | import scala.concurrent.duration.{ Duration, FiniteDuration } 25 | import scala.language.implicitConversions 26 | 27 | trait EigenflowDSL { 28 | 29 | implicit class ExecutionPlanWrapper[A, B](executionPlan: ExecutionPlan[A, B]) { 30 | /** 31 | * Link execution plans. 32 | */ 33 | def ~>[C](nextPlan: ExecutionPlan[B, C]): ExecutionPlan[B, C] = 34 | nextPlan.copy(previous = Some(executionPlan)) 35 | 36 | /** 37 | * Define a simple retry logic, one retry logic for all types of exceptions. 38 | * 39 | * @param interval Interval between retries. 40 | * @param attempts Number of retries. 41 | */ 42 | def retry(interval: FiniteDuration, attempts: Int): ExecutionPlan[A, B] = 43 | executionPlan.copy(recoveryStrategy = _ => Retry(interval, attempts)) 44 | 45 | /** 46 | * Define recovery strategy per exception. 47 | * The "NoRetry" strategy is applied for not covered exceptions, what means fail the process. 48 | * 49 | * @param f Partial function which maps exception to a retry strategy. 50 | */ 51 | def onFailure(f: PartialFunction[Throwable, RecoveryStrategy]): ExecutionPlan[A, B] = 52 | executionPlan.copy(recoveryStrategy = t => (f orElse RecoveryStrategy.default)(t)) 53 | 54 | /** 55 | * Publish custom data to the given topic. 56 | * 57 | * @param f Function to build a key map value from the stage execution result. 58 | */ 59 | def publishMetrics(f: B => Map[String, Double]): ExecutionPlan[A, B] = 60 | executionPlan.copy(publishMetricsMap = Some(f)) 61 | 62 | /** 63 | * Define recovery timeout. Makes sense only in combination with 'onFailure' method. 64 | * 65 | * @param timeout Timeout after which the stage fails if in recovery mode. 66 | */ 67 | def globalRecoveryTimeout(timeout: Duration): ExecutionPlan[A, B] = 68 | executionPlan.copy(recoveryTimeout = Some(timeout.toMillis)) 69 | } 70 | 71 | /** 72 | * A wrapper over JobStage, helps to build DSL where stage logic can be described right beside the stage class, for example: 73 | * 74 | * val downloading = Downloading { 75 | * ... 76 | * } 77 | * 78 | * @param stage Stage to wrap 79 | */ 80 | implicit class ProcessStageWrapper(val stage: ProcessStage) { 81 | /** 82 | * Define a stage execution function for stages w/o input date, usually initial stages. 83 | */ 84 | def apply[B](f: => Future[B])(implicit to: B => String): ExecutionPlan[Unit, B] = 85 | ExecutionPlan(stage, (_: ProcessContext) => _ => f, _ => (), to) 86 | 87 | /** 88 | * Define a stage execution function with the previous stage's result as input. 89 | */ 90 | def apply[A, B](f: A => Future[B])(implicit from: String => A, to: B => String): ExecutionPlan[A, B] = 91 | ExecutionPlan(stage, (_: ProcessContext) => f, from, to) 92 | 93 | /** 94 | * Define a stage execution function with access to the context data. 95 | * The context data usually useful on the very first stage where the access to 'processingDate' makes the most sense. 96 | * 97 | * Other stages (after the first one) should pass context data using stage messages and do not rely on the ProcessContext, 98 | * with a very rare exceptions, see: withContext[A, B] 99 | * 100 | */ 101 | def withContext[B](f: ProcessContext => Future[B])(implicit to: B => String): ExecutionPlan[String, B] = 102 | ExecutionPlan(stage, (data: ProcessContext) => { _: String => f(data) }, x => x, to) 103 | 104 | /** 105 | * Define a stage execution function with access to the run context and the result of previous stage. 106 | * 107 | * This function covers such rare cases when a stage tries to behave differently depending on failure, 108 | * the state can be propagated through the exception and the stage can read it from the context.failure field. 109 | * Note: this technique should be used as the last resort, when it's absolutely necessary to keep state between retries. 110 | * 111 | */ 112 | def withContext[A, B](f: ProcessContext => A => Future[B])(implicit from: String => A, to: B => String): ExecutionPlan[A, B] = 113 | ExecutionPlan(stage, f, from, to) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/environment/ConfigurationLoader.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.environment 18 | 19 | import com.typesafe.config.ConfigFactory 20 | 21 | object ConfigurationLoader { 22 | lazy val config = Option(System.getenv("config")).fold(ConfigFactory.load())(filePath => ConfigFactory.load(filePath)) 23 | } 24 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/helpers/DateHelper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.helpers 18 | 19 | import java.text.SimpleDateFormat 20 | import java.util.Date 21 | 22 | import jawn.ParseException 23 | 24 | object DateHelper { 25 | val SimpleDateFormat = new SimpleDateFormat("yyyyMMdd") 26 | val DateFormat = new SimpleDateFormat("yyyy-MM-dd") 27 | val TimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'") 28 | 29 | SimpleDateFormat.setLenient(false) 30 | DateFormat.setLenient(false) 31 | TimeFormat.setLenient(false) 32 | 33 | def parse(string: String): Option[Date] = { 34 | try { 35 | Some( 36 | if (string.contains("-")) { 37 | if (string.contains("T")) { 38 | TimeFormat.parse(string) 39 | } else { 40 | DateFormat.parse(string) 41 | } 42 | } else { 43 | SimpleDateFormat.parse(string) 44 | }) 45 | } catch { 46 | case e: Throwable => None 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/helpers/PrimitiveImplicits.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.helpers 18 | 19 | import scala.language.implicitConversions 20 | 21 | object PrimitiveImplicits { 22 | implicit def int2String(value: Int): String = value.toString 23 | implicit def string2Int(value: String): Int = Integer.parseInt(value) 24 | implicit def unit2String(value: Unit): String = value.toString 25 | implicit def string2Unit(value: String): Unit = () 26 | } 27 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/process/ProcessFSM.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.process 18 | 19 | import java.util.Date 20 | 21 | import akka.actor.{ Actor, ActorLogging } 22 | import akka.persistence.fsm.PersistentFSM 23 | import com.mediative.eigenflow.StagedProcess 24 | import com.mediative.eigenflow.domain.RecoveryStrategy.{ Fail, Retry } 25 | import com.mediative.eigenflow.domain.fsm._ 26 | import com.mediative.eigenflow.domain.{ ProcessContext, RecoveryStrategy, RetryRegistry } 27 | import com.mediative.eigenflow.process.ProcessFSM.{ CompleteExecution, Continue, FailExecution } 28 | import com.mediative.eigenflow.process.ProcessManager.{ ProcessComplete, ProcessFailed } 29 | import com.mediative.eigenflow.publisher.{ MessagingSystem, ProcessPublisher } 30 | 31 | import scala.concurrent.ExecutionContext.Implicits.global 32 | import scala.concurrent.duration._ 33 | import scala.reflect.ClassTag 34 | import scala.util.{ Failure, Success } 35 | 36 | /** 37 | * State Machine implementation. 38 | * This class persists stage transitions with the process context, thus if the process failed it can be restored 39 | * and start from the failed stage. 40 | * The processingDate parameter is used to restore a process from the previous stage. 41 | * 42 | * @param process Process definition. 43 | * @param processingDate Date the current run applies to. 44 | * @param reset When reset is true the run is forced to start from the Initial stage no matter in what stage it was before. 45 | * 46 | */ 47 | private[eigenflow] class ProcessFSM(process: StagedProcess, 48 | processingDate: Date, 49 | processTypeId: String, 50 | reset: Boolean = false)(implicit override val domainEventClassTag: ClassTag[ProcessEvent], 51 | override val publisher: MessagingSystem) 52 | extends Actor with PersistentFSM[ProcessStage, ProcessContext, ProcessEvent] with ProcessPublisher with ActorLogging { 53 | 54 | // define persistence id depending on the processing date, it makes processes independent of each other. 55 | // it also gives flexibility to redefine processing flow w/o need to clean the history, 56 | // otherwise persistence wouldn't be able to restore the state. 57 | override def persistenceId: String = s"$processTypeId-${processingDate.getTime}" 58 | 59 | override def jobId = processTypeId 60 | 61 | // don't publish messages during recovery! 62 | override def publishingActive = !recoveryRunning 63 | 64 | @throws[Exception](classOf[Exception]) 65 | override def preStart(): Unit = { 66 | buildTransitions(process.executionPlan) 67 | super.preStart() 68 | } 69 | 70 | override protected def onPersistFailure(cause: Throwable, event: Any, seqNr: Long): Unit = { 71 | super.onPersistFailure(cause, event, seqNr) 72 | throw cause 73 | } 74 | 75 | /** 76 | * Update context data and publish events as process moves from stage to stage. 77 | */ 78 | override def applyEvent(domainEvent: ProcessEvent, processContext: ProcessContext): ProcessContext = { 79 | domainEvent match { 80 | 81 | case StageComplete(nextStage, message) => 82 | if (!recoveryRunning) log.info(s"Stage transition: $currentStage ~> $nextStage") 83 | 84 | (currentStage, nextStage) match { 85 | case (Initial, toStage) => 86 | publishProcessStarting(processContext) 87 | publishStageStarting(processContext.processId, nextStage, message) 88 | processContext.updateStage(toStage) 89 | 90 | case (Retrying, _) => 91 | publishStageRetrying(processContext) 92 | processContext 93 | 94 | case (Failed, _) => 95 | publishStageStarting(processContext.processId, nextStage, message) 96 | processContext. 97 | updateStage(nextStage). 98 | emptyRetryRegistry() 99 | 100 | case (fromStage, Complete) => 101 | publishStageComplete(processContext, message) 102 | publishProcessComplete(processContext, process.nextProcessingDate(processContext.processingDate)) 103 | processContext. 104 | updateStage(nextStage). 105 | emptyFailureMessage(). 106 | emptyRetryRegistry() 107 | 108 | case (fromStage, toStage) => 109 | publishStageComplete(processContext, message) 110 | publishStageStarting(processContext.processId, nextStage, message) 111 | processContext. 112 | updateStage(toStage). 113 | updateMessage(message). 114 | emptyFailureMessage(). 115 | emptyRetryRegistry() 116 | } 117 | 118 | case StageRetry(failure, retry, timeout) => 119 | val registeredFailure = RetryRegistry.addFailure(processContext.retryRegistry, failure, retry, timeout) 120 | 121 | if (!recoveryRunning) { 122 | val failureRegistry = RetryRegistry.findFailure(Some(registeredFailure), failure) 123 | failureRegistry.foreach { registry => 124 | log.warning(s"Retry stage $currentStage in ${retry.interval} " + 125 | s"(attempt ${registry.attemptsMade} of ${registry.maxAttempts}). " + 126 | s"Caused by: $failure") 127 | } 128 | } 129 | 130 | processContext. 131 | addFailureMessage(failure). 132 | updateRetryRegistryWith(registeredFailure) 133 | 134 | case StageFailed(failure) => 135 | if (!recoveryRunning) { 136 | log.error(failure, s"Process failed on stage $currentStage. Caused by: $failure") 137 | } 138 | 139 | publishStageFailed(processContext, failure) 140 | publishProcessFailed(processContext, failure) 141 | processContext. 142 | addFailureMessage(failure) 143 | 144 | } 145 | } 146 | 147 | // 148 | // Transition logic from the given stage. 149 | // 150 | startWith(Initial, ProcessContext.default(processingDate)) 151 | 152 | when(Initial) { 153 | case Event(Continue, _) => 154 | moveTo(firstStage) 155 | } 156 | 157 | when(Complete) { 158 | case Event(Continue, _) => 159 | context.parent ! ProcessComplete 160 | stop() 161 | } 162 | 163 | when(Retrying) { 164 | case Event(Continue, processContext) => 165 | moveTo(processContext.stage, processContext.message) 166 | } 167 | 168 | when(Failed) { 169 | case Event(Continue, processContext) => 170 | moveTo(processContext.stage, processContext.message) 171 | } 172 | 173 | override def onRecoveryCompleted(): Unit = { 174 | if (reset) { 175 | startWith(Initial, ProcessContext.default(processingDate)) 176 | } 177 | super.onRecoveryCompleted() 178 | } 179 | 180 | /** 181 | * Register stages based on the given execution plan. 182 | * 183 | */ 184 | private def buildTransitions[A, B](plan: ExecutionPlan[A, B], nextStage: ProcessStage = Complete): Unit = { 185 | when(plan.stage) { 186 | case Event(Continue, processContext) => 187 | try { 188 | plan.f(processContext)(plan.from(processContext.message)).onComplete { 189 | case Success(result) => 190 | plan.publishMetricsMap.foreach(toMap => publishMetrics(processContext, toMap(result))) 191 | self ! CompleteExecution(plan.to(result)) 192 | case Failure(failure: Throwable) => 193 | self ! FailExecution(failure) 194 | } 195 | } catch { 196 | case failure: Throwable => 197 | self ! FailExecution(failure) 198 | } 199 | stay() 200 | case Event(CompleteExecution(message), _) => 201 | moveTo(nextStage, message) 202 | case Event(FailExecution(failure), processContext) => 203 | plan.recoveryStrategy(failure) match { 204 | case retry @ Retry(_, _) => // stage has a recovery plan, check if it's valid for next retry then retry or fail 205 | if (RetryRegistry.isValidForNextRetry(processContext.retryRegistry, failure)) { 206 | retryStage(failure, retry, plan.recoveryTimeout) 207 | } else { 208 | failProcess(failure) 209 | } 210 | case RecoveryStrategy.Complete => // skip this instance and move on (mark as complete) 211 | moveTo(Complete, s"forced to complete by recovery strategy. failure: ${failure.getMessage}") 212 | case Fail => // no recovery plan for this stage, fail the process 213 | failProcess(failure) 214 | } 215 | } 216 | 217 | plan.previous.foreach(buildTransitions(_, plan.stage)) 218 | } 219 | 220 | // 221 | // Helper methods and classes 222 | // 223 | private def moveTo(stage: ProcessStage, message: String = "") = { 224 | goto(stage) applying StageComplete(stage, message) andThen (_ => self ! Continue) 225 | } 226 | 227 | private def failProcess(failure: Throwable) = { 228 | goto(Failed) applying StageFailed(failure) andThen (_ => { 229 | context.parent ! ProcessFailed 230 | context.stop(self) 231 | }) 232 | } 233 | 234 | private def retryStage(failure: Throwable, retry: Retry, timeout: Option[Long]) = { 235 | goto(Retrying) applying StageRetry(failure, retry, timeout) andThen { _ => 236 | val now = System.currentTimeMillis() 237 | val timeDefinedInRetry = now + retry.interval.toMillis 238 | val nextRetryTime = timeout.map { deadline => 239 | timeDefinedInRetry.min(deadline) 240 | }.getOrElse(timeDefinedInRetry) 241 | 242 | val nextRetry = FiniteDuration((nextRetryTime - now) / 1000, SECONDS) 243 | context.system.scheduler.scheduleOnce(nextRetry, self, Continue) 244 | log.debug(s"stage retry scheduled in $nextRetry seconds") 245 | } 246 | } 247 | 248 | private lazy val firstStage: ProcessStage = { 249 | def stageUp(plan: ExecutionPlan[_, _]): ProcessStage = plan.previous.fold(plan.stage)(stageUp) 250 | stageUp(process.executionPlan) 251 | } 252 | 253 | /** 254 | * Alias for the stateName function to make it explicit that it's the current stage. 255 | * 256 | */ 257 | private def currentStage = stateName 258 | 259 | /** 260 | * A context wrapper class for helper functions 261 | */ 262 | private implicit class ProcessContextHelper(processContext: ProcessContext) { 263 | def emptyFailureMessage() = processContext.copy(failure = None) 264 | 265 | def addFailureMessage(failure: Throwable) = processContext.copy(failure = Some(failure)) 266 | 267 | def emptyRetryRegistry() = processContext.copy(retryRegistry = None) 268 | 269 | def updateRetryRegistryWith(retryRegistry: RetryRegistry) = { 270 | processContext.copy(retryRegistry = Some(retryRegistry)) 271 | } 272 | 273 | def updateStage(stage: ProcessStage) = processContext.copy(timestamp = System.currentTimeMillis(), stage = stage) 274 | 275 | def updateMessage(message: String) = processContext.copy(message = message) 276 | } 277 | 278 | } 279 | 280 | object ProcessFSM { 281 | 282 | case object Continue 283 | 284 | // Message to trigger transition after a successful stage execution. 285 | case class CompleteExecution(data: String) 286 | 287 | // Message to trigger recovery or failure after a stage execution failure. 288 | case class FailExecution(failure: Throwable) 289 | 290 | } 291 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/process/ProcessManager.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.process 18 | 19 | import java.util.Date 20 | 21 | import akka.actor.SupervisorStrategy.Escalate 22 | import akka.actor.{ ActorLogging, OneForOneStrategy, Props, SupervisorStrategy } 23 | import akka.persistence.PersistentActor 24 | import com.mediative.eigenflow.StagedProcess 25 | import com.mediative.eigenflow.helpers.DateHelper._ 26 | import com.mediative.eigenflow.publisher.MessagingSystem 27 | 28 | private[eigenflow] object ProcessManager { 29 | 30 | // Persistent Event 31 | case class ProcessingDateState(date: Date, complete: Boolean) 32 | 33 | // Commands 34 | case object Continue 35 | 36 | case object ProcessComplete 37 | 38 | case object ProcessFailed 39 | 40 | val SuccessCode = 0 41 | val FailureCode = 1 42 | } 43 | 44 | /** 45 | * The process parent actor which creates FSM actors which actually run processes based on processingDate. 46 | * 47 | */ 48 | private[eigenflow] class ProcessManager(process: StagedProcess, startDate: Option[Date], processTypeId: String)(implicit val messagingSystem: MessagingSystem) 49 | extends PersistentActor with ActorLogging { 50 | 51 | import com.mediative.eigenflow.process.ProcessManager._ 52 | 53 | override def supervisorStrategy: SupervisorStrategy = { 54 | OneForOneStrategy() { 55 | case t => Escalate 56 | } 57 | } 58 | 59 | override protected def onRecoveryFailure(cause: Throwable, event: Option[Any]): Unit = fail 60 | 61 | override protected def onPersistFailure(cause: Throwable, event: Any, seqNr: Long): Unit = fail 62 | 63 | override def persistenceId: String = s"${processTypeId}-manager" 64 | 65 | override def receiveRecover: Receive = { 66 | case ProcessingDateState(date, complete) => 67 | val dateToProcess = if (complete) { 68 | process.nextProcessingDate(date) // the stage was complete successfully switch to the next processing date 69 | } else { 70 | date 71 | } 72 | context.become(ready(startDate.getOrElse(dateToProcess))) 73 | } 74 | 75 | override def receiveCommand: Receive = ready(startDate.getOrElse(process.initialProcessingDate)) 76 | 77 | def ready(processingDate: Date): Receive = { 78 | case Continue => 79 | val now = new Date 80 | if (processingDate.before(now) || processingDate.equals(now)) { 81 | persist(ProcessingDateState(processingDate, complete = false)) { event => 82 | // TODO: think of a better way to couple `startDate` and `reset` 83 | val processFSM = context.actorOf(Props(new ProcessFSM(process, processingDate, processTypeId, reset = startDate.isDefined))) 84 | 85 | context.become(waitResult(processingDate)) 86 | processFSM ! ProcessFSM.Continue 87 | } 88 | } else { 89 | log.info(s"The process is idle until: ${TimeFormat.format(processingDate)}") 90 | done 91 | } 92 | } 93 | 94 | def waitResult(processingDate: Date): Receive = { 95 | case ProcessComplete => 96 | persist(ProcessingDateState(processingDate, complete = true)) { event => 97 | val nextProcessingDate = process.nextProcessingDate(event.date) 98 | 99 | context.become(ready(nextProcessingDate)) 100 | self ! Continue 101 | } 102 | 103 | case ProcessFailed => 104 | fail 105 | } 106 | 107 | private def fail() = context.parent ! ProcessSupervisor.Failed 108 | private def done() = context.parent ! ProcessSupervisor.Done 109 | } 110 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/process/ProcessSupervisor.scala: -------------------------------------------------------------------------------- 1 | package com.mediative.eigenflow.process 2 | 3 | import java.util.Date 4 | 5 | import akka.actor.SupervisorStrategy.Stop 6 | import akka.actor.{ Actor, ActorRef, OneForOneStrategy, Props, SupervisorStrategy, Terminated } 7 | import com.mediative.eigenflow.StagedProcess 8 | import com.mediative.eigenflow.publisher.MessagingSystem 9 | 10 | object ProcessSupervisor { 11 | def props(process: StagedProcess, date: Option[Date], processTypeId: String, stopSystem: Int => Unit)(implicit messagingSystem: MessagingSystem) = 12 | Props(new ProcessSupervisor(process, date, processTypeId, stopSystem)) 13 | 14 | case object Done 15 | case object Failed 16 | } 17 | 18 | class ProcessSupervisor(val process: StagedProcess, val startDate: Option[Date], processTypeId: String, stopSystem: Int => Unit)(implicit val messagingSystem: MessagingSystem) extends Actor { 19 | private val processManager: ActorRef = context.actorOf(Props(new ProcessManager(process, startDate, processTypeId))) 20 | 21 | context.watch(processManager) 22 | 23 | override def supervisorStrategy: SupervisorStrategy = OneForOneStrategy() { 24 | case _ => Stop 25 | } 26 | 27 | override def receive: Receive = { 28 | case ProcessSupervisor.Done => stopSystem(0) 29 | case ProcessSupervisor.Failed | Terminated(`processManager`) => 30 | stopSystem(1) 31 | case message => processManager.forward(message) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/publisher/MessagingSystem.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.publisher 18 | 19 | import akka.event.LoggingAdapter 20 | import com.typesafe.config.Config 21 | 22 | object MessagingSystem { 23 | def create(config: Config) = { 24 | Class.forName(config.getString("eigenflow.messaging")). 25 | getConstructor(classOf[Config]) 26 | .newInstance(config) 27 | .asInstanceOf[MessagingSystem] 28 | } 29 | } 30 | 31 | abstract class MessagingSystem(config: Config) { 32 | def publish(topic: String, message: String)(implicit log: LoggingAdapter): Unit 33 | def stop(): Unit = () 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/publisher/PrintMessagingSystem.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.publisher 18 | 19 | import akka.event.LoggingAdapter 20 | import com.typesafe.config.Config 21 | 22 | class PrintMessagingSystem(config: Config) extends MessagingSystem(config) { 23 | override def publish(topic: String, message: String)(implicit log: LoggingAdapter): Unit = { 24 | log.info(s"Publishing to $topic :\n$message\n") 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/publisher/ProcessPublisher.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.publisher 18 | 19 | import java.util.Date 20 | 21 | import akka.event.LoggingAdapter 22 | import com.mediative.eigenflow.domain.ProcessContext 23 | import com.mediative.eigenflow.domain.fsm.ProcessStage 24 | import com.mediative.eigenflow.domain.messages._ 25 | import com.mediative.eigenflow.helpers.DateHelper._ 26 | import upickle.default._ 27 | 28 | import scala.language.implicitConversions 29 | 30 | trait ProcessPublisher { 31 | implicit def log: LoggingAdapter 32 | 33 | def publisher: MessagingSystem 34 | 35 | def jobId: String 36 | 37 | // this method is used to forbid publishing during stage recovery to avoid publishing of duplicates or false events. 38 | def publishingActive: Boolean = true 39 | 40 | // high level helper functions 41 | def publishProcessStarting(context: ProcessContext): Unit = { 42 | publishProcessMessage(ProcessMessage( 43 | timestamp = System.currentTimeMillis(), 44 | jobId = jobId, 45 | processId = context.processId, 46 | processingDate = context.processingDate, 47 | state = Processing, 48 | duration = 0, 49 | message = "" 50 | )) 51 | } 52 | 53 | def publishProcessComplete(context: ProcessContext, 54 | nextProcessingDate: Date): Unit = { 55 | val now = System.currentTimeMillis() 56 | publishProcessMessage(ProcessMessage( 57 | timestamp = now, 58 | jobId = jobId, 59 | processId = context.processId, 60 | processingDate = context.processingDate, 61 | state = Complete, 62 | duration = now - context.startTime, 63 | message = nextProcessingDate 64 | )) 65 | } 66 | 67 | def publishProcessFailed(context: ProcessContext, failure: Throwable): Unit = { 68 | publishProcessMessage(ProcessMessage( 69 | timestamp = System.currentTimeMillis(), 70 | jobId = jobId, 71 | processId = context.processId, 72 | processingDate = context.processingDate, 73 | state = Failed, 74 | duration = 0, 75 | message = failure 76 | )) 77 | } 78 | 79 | def publishStageStarting(processId: String, stageWillStart: ProcessStage, message: String): Unit = { 80 | publishStageMessage(StageMessage( 81 | timestamp = System.currentTimeMillis(), 82 | jobId = jobId, 83 | processId = processId, 84 | stage = stageWillStart, 85 | state = Processing, 86 | duration = 0, 87 | message = message 88 | )) 89 | } 90 | 91 | def publishStageComplete(context: ProcessContext, message: String): Unit = { 92 | val now = System.currentTimeMillis() 93 | publishStageMessage(StageMessage( 94 | timestamp = now, 95 | jobId = jobId, 96 | processId = context.processId, 97 | stage = context.stage, 98 | state = Complete, 99 | duration = now - context.startTime, 100 | message = message 101 | )) 102 | } 103 | 104 | def publishStageRetrying(context: ProcessContext): Unit = { 105 | publishStageMessage(StageMessage( 106 | timestamp = System.currentTimeMillis(), 107 | jobId = jobId, 108 | processId = context.processId, 109 | stage = context.stage, 110 | state = Retrying, 111 | duration = 0, 112 | message = "" 113 | )) 114 | } 115 | 116 | def publishStageFailed(context: ProcessContext, failure: Throwable): Unit = { 117 | publishStageMessage(StageMessage( 118 | timestamp = System.currentTimeMillis(), 119 | jobId = jobId, 120 | processId = context.processId, 121 | stage = context.stage, 122 | state = Failed, 123 | duration = 0, 124 | message = failure 125 | )) 126 | } 127 | 128 | def publishMetrics(context: ProcessContext, message: Map[String, Double]): Unit = { 129 | publishMetricsMessage(MetricsMessage( 130 | timestamp = System.currentTimeMillis(), 131 | jobId = jobId, 132 | processId = context.processId, 133 | stage = context.stage, 134 | message = message 135 | )) 136 | } 137 | 138 | // low level helper functions 139 | private def publishProcessMessage(message: ProcessMessage): Unit = { 140 | if (publishingActive) { 141 | publisher.publish("jobs", message) 142 | } 143 | } 144 | 145 | private def publishStageMessage(message: StageMessage): Unit = { 146 | if (publishingActive) { 147 | publisher.publish("stages", message) 148 | } 149 | } 150 | 151 | private def publishMetricsMessage(message: MetricsMessage): Unit = { 152 | if (publishingActive) { 153 | publisher.publish("metrics", message) 154 | } 155 | } 156 | 157 | // implicits 158 | private implicit def processMessageToString(message: ProcessMessage): String = write[ProcessMessage](message) 159 | 160 | private implicit def stageMessageToString(message: StageMessage): String = write[StageMessage](message) 161 | 162 | private implicit def metricsMessageToString(message: MetricsMessage): String = write[MetricsMessage](message) 163 | 164 | implicit def dateToString(date: Date): String = TimeFormat.format(date) 165 | 166 | implicit def stateRecordToString(state: StateRecord): String = state.identifier 167 | 168 | implicit def stageToString(stage: ProcessStage): String = stage.identifier 169 | 170 | implicit def throwableToString(t: Throwable): String = s"${t.getClass.getName}: ${t.getMessage}" 171 | } 172 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/publisher/kafka/KafkaConfiguration.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.publisher.kafka 18 | 19 | import java.util.Properties 20 | 21 | import com.mediative.eigenflow.environment.ConfigurationLoader 22 | import com.typesafe.config.Config 23 | 24 | object KafkaConfiguration { 25 | def properties(config: Config): Properties = { 26 | val propertiesKeys = Seq( 27 | "bootstrap.servers", 28 | "acks", 29 | "retries", 30 | "batch.size", 31 | "linger.ms", 32 | "buffer.memory", 33 | "key.serializer", 34 | "value.serializer", 35 | "topic.prefix") 36 | 37 | val properties = new Properties() 38 | propertiesKeys.foreach(key => properties.setProperty(key, config.getString(s"eigenflow.kafka.${key}"))) 39 | 40 | properties 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/scala/com.mediative.eigenflow/publisher/kafka/KafkaMessagingSystem.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.publisher.kafka 18 | 19 | import akka.event.LoggingAdapter 20 | import com.mediative.eigenflow.publisher.MessagingSystem 21 | import com.typesafe.config.Config 22 | import org.apache.kafka.clients.producer.{ Callback, KafkaProducer, ProducerRecord, RecordMetadata } 23 | 24 | class KafkaMessagingSystem(config: Config) extends MessagingSystem(config) { 25 | private val properties = KafkaConfiguration.properties(config) 26 | private val producer = new KafkaProducer[String, String](properties) 27 | private val topicPrefix = properties.getProperty("topic.prefix") 28 | 29 | override def publish(topic: String, message: String)(implicit log: LoggingAdapter): Unit = { 30 | val topicName = s"$topicPrefix-$topic" 31 | 32 | log.info(s"Publishing to $topicName :\n$message\n") 33 | 34 | producer.send(new ProducerRecord[String, String](topicName, message), new Callback { 35 | override def onCompletion(metadata: RecordMetadata, exception: Exception): Unit = { 36 | if (exception != null) { 37 | log.error(s"Cannot publish to $topicName. Caused by: ${exception.getMessage}", exception) 38 | } 39 | } 40 | }) 41 | () 42 | } 43 | 44 | override def stop(): Unit = { 45 | producer.close() 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/scala/com.mediative.eigenflow.test/dsl/EigenflowDSLTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.test 18 | package dsl 19 | 20 | import java.io.{ FileNotFoundException, IOException } 21 | 22 | import com.mediative.eigenflow.domain.RecoveryStrategy.{ Fail, Retry } 23 | import com.mediative.eigenflow.dsl.EigenflowDSL 24 | import org.scalatest.FreeSpec 25 | import org.scalatest.concurrent.ScalaFutures 26 | import org.scalatest.prop.GeneratorDrivenPropertyChecks 27 | 28 | import scala.concurrent.duration._ 29 | 30 | class EigenflowDSLTest extends FreeSpec with ScalaFutures with EigenflowDSL with GeneratorDrivenPropertyChecks { 31 | "Eigenflow DSL" - { 32 | "stage { logic }" - { 33 | "must generate correct ExecutionPlan" in { 34 | 35 | val value = randomString 36 | 37 | val executionPlan = Stage1 { 38 | toFuture(value) 39 | } 40 | 41 | whenReady(executionPlan.f(defaultProcessContext)(())) { s => 42 | assert(s == value) 43 | } 44 | 45 | assert(executionPlan.stage == Stage1) 46 | assert(executionPlan.previous.isEmpty) 47 | } 48 | } 49 | 50 | "a ~> b" - { 51 | "must link both ExecutionPlan's" in { 52 | val a = Stage2 { 53 | stringFunction 54 | } 55 | 56 | val b = Stage3 { _: String => 57 | stringFunction 58 | } 59 | 60 | val executionPlan = a ~> b 61 | 62 | assert(executionPlan.stage == Stage3) 63 | assert(!executionPlan.previous.isEmpty && executionPlan.previous.get.stage == Stage2) 64 | } 65 | } 66 | 67 | "stage { logic } retry(n, m)" - { 68 | "must register retry strategy in the ExecutionPlan" in { 69 | forAll { (interval: Int, attempts: Int) => 70 | val executionPlan = Stage1 { 71 | stringFunction 72 | } retry (interval.seconds, attempts) 73 | 74 | val strategy = executionPlan.recoveryStrategy(new RuntimeException) 75 | val retry = strategy.asInstanceOf[Retry] 76 | 77 | assert(retry.interval.toSeconds == interval) 78 | assert(retry.attempts == attempts) 79 | } 80 | } 81 | } 82 | 83 | "stage { logic } onFailure { case ... => ... }" - { 84 | "must register retry strategy per exception in the ExecutionPlan" in { 85 | val fileNotFoundAttempts = 3 86 | val ioAttempts = 4 87 | 88 | val executionPlan = Stage1 { 89 | stringFunction 90 | } onFailure { 91 | case _: FileNotFoundException => Retry(3.seconds, fileNotFoundAttempts) 92 | case _: IOException => Retry(3.seconds, ioAttempts) 93 | } 94 | 95 | val fileNotFoundExceptionStrategy = executionPlan.recoveryStrategy(new FileNotFoundException()) 96 | val ioExceptionStrategy = executionPlan.recoveryStrategy(new IOException()) 97 | val unexpectedExceptionStrategy = executionPlan.recoveryStrategy(new RuntimeException()) 98 | 99 | assert(fileNotFoundExceptionStrategy.isInstanceOf[Retry]) 100 | assert(ioExceptionStrategy.isInstanceOf[Retry]) 101 | 102 | val fileNotFoundRetry = fileNotFoundExceptionStrategy.asInstanceOf[Retry] 103 | assert(fileNotFoundRetry.attempts == fileNotFoundAttempts) 104 | 105 | val ioRetry = ioExceptionStrategy.asInstanceOf[Retry] 106 | assert(ioRetry.attempts == ioAttempts) 107 | 108 | unexpectedExceptionStrategy match { 109 | case Fail => assert(true) 110 | case _ => assert(false, "Wrong RecoveryStrategy type assigned by default") 111 | } 112 | } 113 | } 114 | } 115 | 116 | } 117 | -------------------------------------------------------------------------------- /src/test/scala/com.mediative.eigenflow.test/helpers/DateHelperTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow.test.helpers 18 | 19 | import java.text.SimpleDateFormat 20 | import java.util.{ Calendar, Date } 21 | 22 | import com.mediative.eigenflow.helpers.DateHelper 23 | import org.scalatest.FreeSpec 24 | 25 | class DateHelperTest extends FreeSpec { 26 | 27 | val isoDateFormat = "yyyy-MM-dd" 28 | val isoSimpleDateFormat = "yyyyMMdd" 29 | val isoTimeFormat = "yyyy-MM-dd'T'HH:mm:ss'Z'" 30 | 31 | "DateHelper" - { 32 | "when a date is provided" - { 33 | s"must parse $isoDateFormat date properly but without time" in { 34 | testDateFormat(isoDateFormat) 35 | } 36 | 37 | s"must parse $isoSimpleDateFormat date properly but without time" in { 38 | testDateFormat(isoSimpleDateFormat) 39 | } 40 | 41 | "must return None if date format is incorrect" in { 42 | val dateFormat = new SimpleDateFormat("yyyy/MM/dd") 43 | 44 | val date = new Date 45 | 46 | val dateString = dateFormat.format(date) 47 | val result = DateHelper.parse(dateString) 48 | 49 | assert(result.isEmpty) 50 | } 51 | 52 | "must return None if date is invalid" in { 53 | val dates = Seq("2013-12-32", "-2013-12-21", "2013-13-01", "2013-02-30") 54 | 55 | dates.foreach { date => 56 | assert(DateHelper.parse(date).isEmpty) 57 | } 58 | } 59 | } 60 | 61 | "when time is provided" - { 62 | s"must parse date and time in $isoTimeFormat format" in { 63 | val dateFormat = new SimpleDateFormat(isoTimeFormat) 64 | 65 | val date = new Date 66 | 67 | val dateString = dateFormat.format(date) 68 | val resultOption = DateHelper.parse(dateString) 69 | 70 | resultOption.fold(fail(s"failed to parse: ${dateString}")) { result => 71 | assert(dateFormat.format(result) == dateFormat.format(date)) 72 | } 73 | } 74 | } 75 | } 76 | 77 | private def testDateFormat(format: String): Unit = { 78 | val dateFormat = new SimpleDateFormat(format) 79 | val date = new Date 80 | 81 | val dateString = dateFormat.format(date) 82 | 83 | val resultOption = DateHelper.parse(dateString) 84 | 85 | resultOption.fold(fail(s"failed to parse: ${dateString}")) { result => 86 | val calendar = Calendar.getInstance() 87 | calendar.setTime(result) 88 | assert(calendar.get(Calendar.HOUR) == 0) 89 | assert(calendar.get(Calendar.MINUTE) == 0) 90 | assert(dateFormat.format(result) == dateFormat.format(date)) 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/test/scala/com.mediative.eigenflow.test/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Mediative 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mediative.eigenflow 18 | 19 | import java.util.{ Date, UUID } 20 | 21 | import com.mediative.eigenflow.domain.ProcessContext 22 | import com.mediative.eigenflow.domain.RecoveryStrategy.Complete 23 | import com.mediative.eigenflow.domain.fsm.{ ExecutionPlan, Initial, ProcessStage } 24 | 25 | import scala.concurrent.ExecutionContext.Implicits.global 26 | import scala.concurrent.Future 27 | import scala.concurrent.duration._ 28 | 29 | package object test { 30 | 31 | case object Stage1 extends ProcessStage 32 | 33 | case object Stage2 extends ProcessStage 34 | 35 | case object Stage3 extends ProcessStage 36 | 37 | def defaultProcessContext: ProcessContext = { 38 | ProcessContext(timestamp = System.currentTimeMillis(), 39 | processingDate = new Date(), 40 | processId = UUID.randomUUID().toString, 41 | startTime = System.currentTimeMillis(), 42 | stage = Initial, 43 | message = "") 44 | } 45 | 46 | def randomString: String = UUID.randomUUID().toString 47 | 48 | def stringFunction: Future[String] = { 49 | Future { 50 | randomString 51 | } 52 | } 53 | 54 | def toFuture[A](a: A): Future[A] = { 55 | Future { 56 | a 57 | } 58 | } 59 | 60 | val `Stage1 ~> Stage2` = new StagedProcess { 61 | val a = Stage1 { 62 | stringFunction 63 | } 64 | val b = Stage2 { _: String => 65 | stringFunction 66 | } 67 | 68 | override def executionPlan: ExecutionPlan[_, _] = a ~> b 69 | 70 | override def nextProcessingDate(lastCompleted: Date): Date = new Date() 71 | } 72 | 73 | val `Stage1(retry) ~> Stage2` = new StagedProcess { 74 | val a = Stage1 withContext { ctx: ProcessContext => 75 | if (ctx.failure.isEmpty) { 76 | throw new RuntimeException 77 | } 78 | stringFunction 79 | } retry (10.milliseconds, 1) 80 | 81 | val b = Stage2 { _: String => 82 | stringFunction 83 | } 84 | 85 | override def executionPlan: ExecutionPlan[_, _] = a ~> b 86 | 87 | override def nextProcessingDate(lastCompleted: Date): Date = new Date() 88 | } 89 | 90 | val `Stage1(Complete) ~> Stage2` = new StagedProcess { 91 | val a = Stage1 withContext { ctx: ProcessContext => 92 | if (ctx.failure.isEmpty) { 93 | throw new RuntimeException 94 | } 95 | stringFunction 96 | } onFailure { 97 | case _: Throwable => Complete 98 | } 99 | 100 | val b = Stage2 { _: String => 101 | stringFunction 102 | } 103 | 104 | override def executionPlan: ExecutionPlan[_, _] = a ~> b 105 | 106 | override def nextProcessingDate(lastCompleted: Date): Date = new Date() 107 | } 108 | 109 | val `Stage1 ~> Stage2(FailOnce) ~> Stage3` = new StagedProcess { 110 | val a = Stage1 withContext { ctx: ProcessContext => 111 | stringFunction 112 | } onFailure { 113 | case _: Throwable => Complete 114 | } 115 | 116 | val b = Stage2 withContext { ctx: ProcessContext => 117 | { _: String => 118 | if (ctx.failure.isEmpty) { 119 | throw new RuntimeException 120 | } 121 | stringFunction 122 | } 123 | } 124 | 125 | val c = Stage3 { _: String => 126 | stringFunction 127 | } 128 | 129 | override def executionPlan: ExecutionPlan[_, _] = a ~> b 130 | 131 | override def nextProcessingDate(lastCompleted: Date): Date = new Date() 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/test/scala/com.mediative.eigenflow.test/publisher/ProcessPublisherTest.scala: -------------------------------------------------------------------------------- 1 | package com.mediative.eigenflow.test.publisher 2 | 3 | import java.util.Date 4 | 5 | import akka.event.LoggingAdapter 6 | import com.mediative.eigenflow.domain.ProcessContext 7 | import com.mediative.eigenflow.domain.messages._ 8 | import com.mediative.eigenflow.helpers.DateHelper._ 9 | import com.mediative.eigenflow.publisher.{ MessagingSystem, ProcessPublisher } 10 | import com.typesafe.config.ConfigFactory 11 | import org.scalatest.FreeSpec 12 | import org.scalatest.mock.MockitoSugar 13 | import upickle.default._ 14 | 15 | import scala.collection.mutable 16 | import scala.language.{ implicitConversions, reflectiveCalls } 17 | 18 | class ProcessPublisherTest extends FreeSpec { 19 | 20 | class TestContext { 21 | val processContext = ProcessContext.default(new Date(0)) 22 | 23 | val mockMessagingSystem = new MessagingSystem(ConfigFactory.empty()) { 24 | val publishedMessages = mutable.ArrayBuffer[(String, String)]() 25 | 26 | override def publish(topic: String, message: String)(implicit log: LoggingAdapter): Unit = { 27 | publishedMessages.append(topic -> message) 28 | } 29 | } 30 | 31 | val mockLog = new LoggingAdapter { 32 | override protected def notifyInfo(message: String): Unit = {} 33 | override def isErrorEnabled: Boolean = false 34 | override def isInfoEnabled: Boolean = false 35 | override def isDebugEnabled: Boolean = false 36 | override protected def notifyError(message: String): Unit = {} 37 | override protected def notifyError(cause: Throwable, message: String): Unit = {} 38 | override def isWarningEnabled: Boolean = false 39 | override protected def notifyWarning(message: String): Unit = {} 40 | override protected def notifyDebug(message: String): Unit = {} 41 | } 42 | 43 | val publisher = new ProcessPublisher { 44 | val log = mockLog 45 | override def jobId: String = "TestJob" 46 | 47 | override def publisher: MessagingSystem = mockMessagingSystem 48 | } 49 | } 50 | 51 | def context(f: (TestContext) => Unit): Unit = { 52 | f(new TestContext) 53 | } 54 | 55 | def defaultProcessMessage(context: TestContext): ProcessMessage = { 56 | ProcessMessage( 57 | timestamp = 0, 58 | jobId = context.publisher.jobId, 59 | processId = context.processContext.processId, 60 | processingDate = TimeFormat.format(context.processContext.processingDate), 61 | state = Processing.identifier, 62 | duration = 0, 63 | message = "" 64 | ) 65 | } 66 | 67 | def defaultStageMessage(context: TestContext): StageMessage = { 68 | StageMessage( 69 | timestamp = 0, 70 | jobId = context.publisher.jobId, 71 | processId = context.processContext.processId, 72 | stage = context.processContext.stage.identifier, 73 | state = Processing.identifier, 74 | duration = 0, 75 | message = "" 76 | ) 77 | } 78 | 79 | implicit class ProcessMessageHelper(message: ProcessMessage) { 80 | def ignoreTimeFields(): ProcessMessage = message.copy(timestamp = 0, duration = 0) 81 | } 82 | implicit class StageMessageHelper(message: StageMessage) { 83 | def ignoreTimeFields(): StageMessage = message.copy(timestamp = 0, duration = 0) 84 | } 85 | implicit class MetricsMessageHelper(message: MetricsMessage) { 86 | def ignoreTimeFields(): MetricsMessage = message.copy(timestamp = 0) 87 | } 88 | 89 | "publishProcessStarting" - { 90 | "publish the expected message" in context { context => 91 | context.publisher.publishProcessStarting(context.processContext) 92 | 93 | val expectedMessage = defaultProcessMessage(context).copy( 94 | state = Processing.identifier 95 | ) 96 | 97 | val actualMessages = context.mockMessagingSystem.publishedMessages.map { 98 | case (key, value) => key -> read[ProcessMessage](value).ignoreTimeFields() 99 | } 100 | 101 | assert(actualMessages === List("jobs" -> expectedMessage)) 102 | } 103 | } 104 | "publishProcessComplete" - { 105 | "publish the expected message" in context { context => 106 | val nextProcessingDate = new Date() 107 | context.publisher.publishProcessComplete(context.processContext, nextProcessingDate) 108 | 109 | val expectedMessage = defaultProcessMessage(context).copy( 110 | state = Complete.identifier, 111 | message = TimeFormat.format(nextProcessingDate) 112 | ) 113 | 114 | val actualMessages = context.mockMessagingSystem.publishedMessages.map { 115 | case (key, value) => key -> read[ProcessMessage](value).ignoreTimeFields() 116 | } 117 | 118 | assert(actualMessages === List("jobs" -> expectedMessage)) 119 | } 120 | } 121 | "publishProcessFailed" - { 122 | "publish the expected message" in context { context => 123 | val failure = new RuntimeException("Boom") 124 | 125 | context.publisher.publishProcessFailed(context.processContext, failure) 126 | 127 | val expectedMessage = defaultProcessMessage(context).copy( 128 | state = Failed.identifier, 129 | message = s"${failure.getClass.getName}: ${failure.getMessage}" 130 | ) 131 | 132 | val actualMessages = context.mockMessagingSystem.publishedMessages.map { 133 | case (key, value) => key -> read[ProcessMessage](value).ignoreTimeFields() 134 | } 135 | 136 | assert(actualMessages === List("jobs" -> expectedMessage)) 137 | } 138 | } 139 | "publishStageStarting" - { 140 | "publish the expected message" in context { context => 141 | val message = "Some Message" 142 | 143 | context.publisher.publishStageStarting(context.processContext.processId, context.processContext.stage, message) 144 | 145 | val expectedMessage = defaultStageMessage(context).copy( 146 | state = Processing.identifier, 147 | message = message 148 | ) 149 | 150 | val actualMessages = context.mockMessagingSystem.publishedMessages.map { 151 | case (key, value) => key -> read[StageMessage](value).ignoreTimeFields() 152 | } 153 | 154 | assert(actualMessages === List("stages" -> expectedMessage)) 155 | } 156 | } 157 | "publishStageComplete" - { 158 | "publish the expected message" in context { context => 159 | val message = "Some Message" 160 | 161 | context.publisher.publishStageComplete(context.processContext, message) 162 | 163 | val expectedMessage = defaultStageMessage(context).copy( 164 | state = Complete.identifier, 165 | message = message 166 | ) 167 | 168 | val actualMessages = context.mockMessagingSystem.publishedMessages.map { 169 | case (key, value) => key -> read[StageMessage](value).ignoreTimeFields() 170 | } 171 | 172 | assert(actualMessages === List("stages" -> expectedMessage)) 173 | } 174 | } 175 | "publishStageRetrying" - { 176 | "publish the expected message" in context { context => 177 | context.publisher.publishStageRetrying(context.processContext) 178 | 179 | val expectedMessage = defaultStageMessage(context).copy( 180 | state = Retrying.identifier, 181 | message = context.processContext.message 182 | ) 183 | 184 | val actualMessages = context.mockMessagingSystem.publishedMessages.map { 185 | case (key, value) => key -> read[StageMessage](value).ignoreTimeFields() 186 | } 187 | 188 | assert(actualMessages === List("stages" -> expectedMessage)) 189 | } 190 | } 191 | "publishStageFailed" - { 192 | "publish the expected message" in context { context => 193 | val failure = new RuntimeException("Boom") 194 | 195 | context.publisher.publishStageFailed(context.processContext, failure) 196 | 197 | val expectedMessage = defaultStageMessage(context).copy( 198 | state = Failed.identifier, 199 | message = s"${failure.getClass.getName}: ${failure.getMessage}" 200 | ) 201 | 202 | val actualMessages = context.mockMessagingSystem.publishedMessages.map { 203 | case (key, value) => key -> read[StageMessage](value).ignoreTimeFields() 204 | } 205 | 206 | assert(actualMessages === List("stages" -> expectedMessage)) 207 | } 208 | } 209 | "publishMetrics" - { 210 | "publish the expected message" in context { context => 211 | val metricType = "Type" 212 | 213 | val message = Map( 214 | "k1" -> 1.0d, 215 | "k2" -> 2.0d, 216 | "k2" -> 3.0d 217 | ) 218 | 219 | context.publisher.publishMetrics(context.processContext, message) 220 | 221 | val expectedMessage = MetricsMessage( 222 | timestamp = 0, 223 | jobId = context.publisher.jobId, 224 | processId = context.processContext.processId, 225 | stage = context.processContext.stage.identifier, 226 | message = message) 227 | 228 | val actualMessages = context.mockMessagingSystem.publishedMessages.map { 229 | case (key, value) => key -> read[MetricsMessage](value).ignoreTimeFields() 230 | } 231 | 232 | assert(actualMessages === List("metrics" -> expectedMessage)) 233 | } 234 | } 235 | 236 | } 237 | -------------------------------------------------------------------------------- /src/test/scala/com/mediative/eigenflow/publisher/MessagingSystemTest.scala: -------------------------------------------------------------------------------- 1 | package com.mediative.eigenflow.publisher 2 | 3 | import com.typesafe.config.{ ConfigValueFactory, ConfigObject, ConfigValue, ConfigFactory } 4 | import org.scalatest.FreeSpec 5 | 6 | class MessagingSystemTest extends FreeSpec { 7 | "create" - { 8 | "should create the appropriate messaging config" in { 9 | val config = ConfigFactory.empty().withValue("eigenflow.messaging", ConfigValueFactory.fromAnyRef("com.mediative.eigenflow.publisher.PrintMessagingSystem")) 10 | 11 | val messagingSystem = MessagingSystem.create(config) 12 | 13 | assert(messagingSystem.isInstanceOf[PrintMessagingSystem]) 14 | } 15 | "should fail if the messaging system does not exist." in { 16 | val config = ConfigFactory.empty().withValue("eigenflow.messaging", ConfigValueFactory.fromAnyRef("Doesn't Exist")) 17 | 18 | val _ = intercept[ClassNotFoundException] { 19 | MessagingSystem.create(config) 20 | } 21 | 22 | } 23 | } 24 | } 25 | --------------------------------------------------------------------------------