├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── custom.md │ └── feature_request.md └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── .rspec ├── .ruby-gemset ├── .ruby-version ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Gemfile ├── Gemfile.lock ├── LICENSE.txt ├── README.md ├── Rakefile ├── bin ├── console └── setup ├── lib ├── taskinator.rb └── taskinator │ ├── api.rb │ ├── builder.rb │ ├── complete_on.rb │ ├── create_process_worker.rb │ ├── definition.rb │ ├── executor.rb │ ├── instrumentation.rb │ ├── logger.rb │ ├── persistence.rb │ ├── process.rb │ ├── queues.rb │ ├── queues │ ├── active_job.rb │ ├── delayed_job.rb │ ├── resque.rb │ └── sidekiq.rb │ ├── redis_connection.rb │ ├── task.rb │ ├── task_worker.rb │ ├── tasks.rb │ ├── version.rb │ ├── visitor.rb │ └── workflow.rb ├── processes_workflow.png ├── sequence.txt ├── spec ├── examples │ ├── process_examples.rb │ ├── queue_adapter_examples.rb │ └── task_examples.rb ├── spec_helper.rb ├── support │ ├── delayed_job.rb │ ├── mock_definition.rb │ ├── mock_model.rb │ ├── process_methods.rb │ ├── sidekiq_matchers.rb │ ├── spec_support.rb │ ├── task_methods.rb │ ├── test_definition.rb │ ├── test_flow.rb │ ├── test_flows.rb │ ├── test_instrumenter.rb │ ├── test_job.rb │ ├── test_job_task.rb │ ├── test_process.rb │ ├── test_queue.rb │ ├── test_step_task.rb │ ├── test_subprocess_task.rb │ └── test_task.rb └── taskinator │ ├── api_spec.rb │ ├── builder_spec.rb │ ├── complex_process_spec.rb │ ├── create_process_worker_spec.rb │ ├── definition_spec.rb │ ├── executor_spec.rb │ ├── instrumentation_spec.rb │ ├── persistence_spec.rb │ ├── process_spec.rb │ ├── queues │ ├── active_job_spec.rb │ ├── delayed_job_spec.rb │ ├── resque_spec.rb │ ├── sidekiq_spec.rb │ └── test_queue_adapter_spec.rb │ ├── queues_spec.rb │ ├── task_spec.rb │ ├── task_worker_spec.rb │ ├── taskinator_spec.rb │ ├── tasks_spec.rb │ ├── test_flows_spec.rb │ └── visitor_spec.rb ├── taskinator.gemspec └── tasks_workflow.png /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/custom.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Custom issue template 3 | about: Describe this issue template's purpose here. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | 7 | pull_request: 8 | branches: [ master ] 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | ruby-version: 16 | - '2.7' 17 | - '3.0' 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Set up Ruby 21 | uses: ruby/setup-ruby@v1 22 | with: 23 | ruby-version: ${{ matrix.ruby-version }} 24 | bundler-cache: true 25 | - name: Build Gem 26 | run: bundle exec rake build 27 | - name: Run tests 28 | run: bundle exec rake spec 29 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | release: 5 | types: [ published ] 6 | 7 | jobs: 8 | publish: 9 | if: ${{ github.event_name == 'release' && github.event.action == 'published' }} 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - name: Release Gem 14 | uses: virtualstaticvoid/publish-rubygems-action@v4 15 | env: 16 | GIT_NAME: "Chris Stefano" 17 | GIT_EMAIL: "virtualstaticvoid@gmail.com" 18 | RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}} 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | InstalledFiles 7 | _yardoc 8 | coverage 9 | doc/ 10 | lib/bundler/man 11 | pkg 12 | rdoc 13 | spec/reports 14 | test/tmp 15 | test/version_tmp 16 | tmp 17 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format documentation 3 | -------------------------------------------------------------------------------- /.ruby-gemset: -------------------------------------------------------------------------------- 1 | taskinator 2 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | ruby-3.0.0 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | v?.?.? - ?? ??? ???? 2 | --- 3 | 4 | v0.5.2 - 04 Oct 2024 5 | --- 6 | * Time arguments fix for Redis 5.0. Fixes #28 7 | 8 | v0.5.1 - 06 Jan 2023 9 | --- 10 | * Include process definition in processes, tasks and payloads to aid debugging. 11 | * Increased test coverage for process and task specs. 12 | * Removed `statsd` instrumentation. 13 | * Removed unused `Taskinator::Visitor::XmlVisitor` implementation. 14 | * Various refactorings and clean ups. 15 | * Bug fixes for process class when used as a sub-process 16 | * Add handling for unknown types when deserializing old processes 17 | * Raises `UnknownTypeError` when trying to invoke processes or Jobs of unknown types 18 | 19 | v0.5.0 - 18 Feb 2022 20 | --- 21 | * Removed unused `ProcessWorker` class and related queue methods. 22 | * Refactored `TestQueueAdapter` to correctly implement queue adapter for use in specs. 23 | * Added deprecation for `Taskinator::Process::Concurrent#concurrency_method` option. 24 | 25 | v0.4.7 - 17 Feb 2022 26 | --- 27 | * Use newer format for `pipelined` and `multi` requests in Redis. 28 | 29 | v0.4.6 - 12 Feb 2022 30 | --- 31 | * Upgrade actionpack for [information vulnerability fix](https://github.com/virtualstaticvoid/taskinator/security/dependabot/3). 32 | 33 | v0.4.5 - 30 Jan 2022 34 | --- 35 | * Upgrade sidekiq dependency for [CVE-2022-23837](https://github.com/advisories/GHSA-jrfj-98qg-qjgv). 36 | 37 | v0.4.4 - 17 Jan 2022 38 | --- 39 | * Add support for `ActiveJob`. 40 | 41 | v0.4.3 - 14 Jan 2022 42 | --- 43 | * Add `#find_process` and `#find_task` methods to `Taskinator::Api`. 44 | * Bug fix to API when enumerating processes. 45 | * Updated dependencies. 46 | 47 | v0.4.2 - 16 Mar 2021 48 | --- 49 | * Bug fix for process/task keys not expired upon completion. 50 | 51 | v0.4.1 - 15 Mar 2021 52 | --- 53 | * Optimisation to exclude sub-processes which don't have any tasks. 54 | * Preparations for upgrade to Ruby 3 and ActiveSupport 6 55 | 56 | v0.4.0 - 4 Mar 2021 57 | --- 58 | * Bug fix `job` tasks which have no arguments to the `perform` method. 59 | * Added support for having `perform` method as a class method. 60 | 61 | v0.3.16 - 17 Feb 2021 62 | --- 63 | * Bug fix to decrement pending counts for sequential tasks. 64 | * Bug fix to allow concurrent tasks to be retried (via Resque) and to complete processes. 65 | 66 | v0.3.15 - 22 Nov 2018 67 | --- 68 | * Updated dependencies. 69 | 70 | v0.3.14 - 13 Jul 2018 71 | --- 72 | * Updated dependencies. 73 | * Removed gemnasium. 74 | 75 | v0.3.13 - 23 Sep 2017 76 | --- 77 | * Updated dependencies. 78 | 79 | v0.3.12 - 23 Sep 2017 80 | --- 81 | * Spec fixes. 82 | * Updated dependencies. 83 | 84 | v0.3.11 - 1 Nov 2016 85 | --- 86 | * Removed `redis-semaphore` gem and use INCRBY to track pending concurrent tasks instead. 87 | * Added instrumentation using statsd. 88 | * Bug fixes to key expiry logic. 89 | * Refactored process and task state transistions. 90 | 91 | v0.3.10 - 1 Nov 2016 92 | --- 93 | * Added support for serializing to XML. 94 | * Improvements to process and task states. 95 | 96 | v0.3.9 - 12 Sep 2016 97 | --- 98 | * Added benchmark for redis-mutex. 99 | 100 | v0.3.7 - 18 Aug 2016 101 | --- 102 | * Bug fix to `option?` method. 103 | 104 | v0.3.6 - 11 Nov 2015 105 | --- 106 | * Added visitor for performing clean up of completed processes/tasks. 107 | * Performance improvement to instrumentation payload; removed references to task/process and use intrinsic types. 108 | * Clean up of keys, via `cleanup` method use key expiry. 109 | 110 | v0.3.5 - 02 Nov 2015 111 | --- 112 | * Updated the keys used when persisting processes and tasks in Redis, so they fall in the same key space. 113 | * Added clean up code to remove data from Redis when a process completes. 114 | * Introduced `Taskinator.generate_uuid` method 115 | * Use Redis pipelined mode to persist processes and tasks. 116 | * Added warning output to log if serialized arguments are bigger than 2MB. 117 | * Introduced scoping for keys in Redis in order to better support multi-tenancy requirements. 118 | * Added XmlVisitor for extracting processes/tasks into XML. 119 | * Introduced `ProcessWorker` (incomplete) which will be used to incrementally build sub-process in order to speed up overall processing for big processes. 120 | 121 | v0.3.3 - 29 Oct 2015 122 | --- 123 | * Bug fix for options handling when defining processes using `define_concurrent_process`. 124 | 125 | v0.3.2 - 18 Sep 2015 126 | --- 127 | * Bug fix to argument handling when using `create_process_remotely` method. 128 | 129 | v0.3.1 - 16 Sep 2015 130 | --- 131 | * Added redis-semaphore gem, for fix to concurrent processes completion logic. 132 | 133 | v0.3.0 - 28 Aug 2015 134 | --- 135 | * Added created_at and updated_at to process and task as attributes. 136 | * Improved serialization visitor to include an optional converter block for deserialization of attribute values. 137 | * Corrections to lazy loader logic and speed improvements. 138 | * Removed JobWorker as it's no longer necessary. 139 | * Improvements to instrumentation. 140 | * Removed workflow gem, and refactored process and task to implement the basics instead. 141 | * Several bug fixes. 142 | 143 | v0.2.0 - 31 Jul 2015 144 | --- 145 | * Bug fix for `create_process_remotely` so that it returns the process uuid instead of nil. 146 | * Removed reload functionality, since it isn't used anymore 147 | * Added missing instrumentation events for task, job and subprocess completed events. 148 | * Bug fix for when `sequential` or `concurrent` steps don't have any tasks to still continue processing. 149 | * Refactoring to remove dead code and "reload" functionality. 150 | * Improvements to console and rake to use console instrumenter. 151 | * Consolidation of instrumentation events. Added `type` to payload. 152 | * Improvements to error handling. 153 | 154 | v0.1.1 - 23 Jul 2015 [Yanked] 155 | --- 156 | * Bug fix for option parameter handling. 157 | 158 | v0.1.0 - 23 Jul 2015 [Yanked] 159 | --- 160 | * Fixed issue with persistence of options passed to `create_process` on the respective `Process` and `Task` instances. 161 | * Improvements to process creation logic. 162 | * Namespaced instrumentation event names. 163 | * Added process completed, cancelled and failed instrumentation events. 164 | * Include additional data in the instrumentation payload. E.g. Process options and percentages. 165 | * Refactored the way processes/tasks get queued, to prevent unnecessary queuing of contained processes/tasks. 166 | * Removed `ProcessWorker` since it isn't needed anymore. 167 | 168 | v0.0.18 - 14 Jul 2015 169 | --- 170 | * Fixed issue with `Taskinator::Api::Processes#each` method, which was causing a Segmentation fault. 171 | * Added statistics information. 172 | * Improved specifications code coverage. 173 | 174 | v0.0.17 - 11 Jul 2015 175 | --- 176 | * Fixed issue with `Taskinator::Task#each` method, which was causing a Segmentation fault. 177 | * Added `define_sequential_process` and `define_concurrent_process` methods for defining processes. 178 | * Added `ConsoleInstrumenter` instrumenter implementation. 179 | * Required `resque` for console and rake tasks, to make debugging easier 180 | 181 | v0.0.16 - 25 Jun 2015 182 | --- 183 | * Added ability to enqueue the creation of processes; added a new worker, `CreateProcessWorker` 184 | * Added support for instrumentation 185 | * Improvements to error handling 186 | * Bug fix for the persistence of the `queue` attribute for `Process` and `Task` 187 | * Code clean up and additional specs added 188 | 189 | v0.0.15 - 28 May 2015 190 | --- 191 | * Added ability to specify the queue to use when enqueing processes, tasks and jobs 192 | * Improvements to specs for testing with sidekiq; added `rspec-sidekiq` as development dependency 193 | * Gem dependencies updated as per Gemnasium advisory 194 | 195 | v0.0.14 - 12 May 2015 196 | --- 197 | * Bug fix for fail! methods 198 | * Bug fix to parameter handling by for_each method 199 | 200 | v0.0.13 - 11 May 2015 201 | --- 202 | * Bug fix to `Taskinator::Api` for listing of processes; should only include top-level processes 203 | * Gem dependencies updated as per Gemnasium advisory 204 | 205 | v0.0.12 - 20 Apr 2015 206 | --- 207 | * Gem dependencies updated as per Gemnasium advisory 208 | 209 | v0.0.11 - 2 Mar 2015 210 | --- 211 | * Gem dependencies updated as per Gemnasium advisory 212 | 213 | v0.0.10 - 26 Feb 2015 214 | --- 215 | * Documentation updates 216 | 217 | v0.0.9 - 19 Dec 2014 218 | --- 219 | * Various bug fixes 220 | * Added error logging 221 | * Workflow states now include `complete` event 222 | * Gem dependencies updated as per Gemnasium advisory 223 | 224 | v0.0.8 - 11 Nov 2014 225 | --- 226 | * Added support for argument chaining with `for_each` and `transform` 227 | * Documentation updates 228 | * Gem dependencies updated as per Gemnasium advisory 229 | 230 | v0.0.7 - 16 Oct 2014 231 | --- 232 | * Added better option handling; introduced `option?(key)` method 233 | * Added support for definining the expected arguments for a process 234 | * Gem dependencies updated as per Gemnasium advisory 235 | 236 | v0.0.5 - 17 Sep 2014 237 | --- 238 | * Various of bug fixes 239 | * Improved error handling 240 | * Added logging for queuing of processes, tasks and jobs 241 | 242 | v0.0.4 - 12 Sep 2014 243 | --- 244 | * Improvements to serialization; make use of GlobalID functionality 245 | * Added support for "job" tasks; reusing existing workers as tasks 246 | 247 | v0.0.3 - 2 Sep 2014 248 | --- 249 | * Added failure steps to workflow of processes and tasks 250 | 251 | v0.0.2 - 12 Aug 2014 252 | --- 253 | * Refactored how tasks are defined in definitions 254 | 255 | v0.0.1 - 12 Aug 2014 256 | --- 257 | * Initial release 258 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | virtualstaticvoid@gmail.com. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | All kinds of contributions are welcome. By participating you agree to follow the [code of conduct]. 4 | 5 | If you find an issue, have an idea for a new feature or any improvement, you can report them 6 | as issues or submit pull requests. To maintain an organized process, please read the following 7 | guidelines before submitting. 8 | 9 | ## Reporting Issues 10 | 11 | Before reporting new issues, please verify if there are any existing issues for the same problem 12 | by searching in [issues]. 13 | 14 | Please make sure to give a clear name to the issue and describe it with all relevant information. 15 | Giving examples can help understand your suggestion or, in case of issues to reproduce the problem. 16 | 17 | ## Sending Pull Requests 18 | 19 | Look at existing [issues] to see if there are any related issues that your feature/fix should 20 | consider. If there are none, please create one describing your implementations intent. You should 21 | always mention a related issue in the pull request description. 22 | 23 | While writing code, follow the code conventions that you find in the existing code. 24 | 25 | Try to write short, clear and objective commit messages too. You can squash your commits and 26 | improve your commit messages once your done. 27 | 28 | Also make sure to add good tests to your new code. Only refactoring of tested features and 29 | documentation do not need new tests. This way your changes will be documented and future 30 | changes will not break what you added. 31 | 32 | ### Step-by-step 33 | 34 | - Fork the repository. 35 | - Commit and push until your are happy of what you have done. 36 | - Execute the full test suite to ensure all is passing. 37 | - Squash commits if necessary. 38 | - Push to your repository. 39 | - Open a pull request to the original repository. 40 | - Give your pull request a good description. Do not forget to mention a related issue. 41 | 42 | ## Running Tests 43 | 44 | Once you cloned the gem to your development environment, you should run the following command 45 | from the root folder to install the project dependencies. 46 | 47 | ```bash 48 | ./bin/setup 49 | ``` 50 | 51 | After that, you can execute the tests suite by running the following command from the root folder. 52 | 53 | ```bash 54 | rake spec 55 | ``` 56 | 57 | [code of conduct]: CODE_OF_CONDUCT.md 58 | [issues]: https://github.com/virtualstaticvoid/taskinator/issues 59 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in Taskinator.gemspec 4 | gemspec 5 | 6 | # queues 7 | gem 'activejob' , '>= 5.0.0' 8 | gem 'sidekiq' , '>= 3.5.0', '< 7.0.0' 9 | gem 'rspec-sidekiq' , '>= 2.1.0' 10 | 11 | gem 'delayed_job' , '~> 4.1.0' 12 | 13 | gem 'resque' , '>= 1.25.2' 14 | gem 'resque_spec' , '>= 0.16.0' 15 | 16 | # other 17 | gem 'bundler' , '>= 1.6.0' 18 | gem 'rake' , '>= 10.3.0' 19 | gem 'activesupport' , '~> 5.2.0' 20 | 21 | gem 'rspec' 22 | gem 'rspec-rails' , '>= 2.0' 23 | 24 | gem 'coveralls' , '>= 0.8.22' 25 | gem 'pry' , '>= 0.9.0' 26 | gem 'pry-byebug' , '>= 1.3.0' 27 | 28 | gem 'fakeredis' , '~> 0.7.0' 29 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | taskinator (0.5.2) 5 | builder (>= 3.2.2) 6 | connection_pool (>= 2.2.0) 7 | globalid (>= 0.3) 8 | json (>= 1.8.2) 9 | redis (>= 3.2.1) 10 | redis-namespace (>= 1.5.2) 11 | thwait (>= 0.2) 12 | 13 | GEM 14 | remote: https://rubygems.org/ 15 | specs: 16 | actionpack (5.2.8.1) 17 | actionview (= 5.2.8.1) 18 | activesupport (= 5.2.8.1) 19 | rack (~> 2.0, >= 2.0.8) 20 | rack-test (>= 0.6.3) 21 | rails-dom-testing (~> 2.0) 22 | rails-html-sanitizer (~> 1.0, >= 1.0.2) 23 | actionview (5.2.8.1) 24 | activesupport (= 5.2.8.1) 25 | builder (~> 3.1) 26 | erubi (~> 1.4) 27 | rails-dom-testing (~> 2.0) 28 | rails-html-sanitizer (~> 1.0, >= 1.0.3) 29 | activejob (5.2.8.1) 30 | activesupport (= 5.2.8.1) 31 | globalid (>= 0.3.6) 32 | activesupport (5.2.8.1) 33 | concurrent-ruby (~> 1.0, >= 1.0.2) 34 | i18n (>= 0.7, < 2) 35 | minitest (~> 5.1) 36 | tzinfo (~> 1.1) 37 | builder (3.2.4) 38 | byebug (11.1.3) 39 | coderay (1.1.3) 40 | concurrent-ruby (1.1.10) 41 | connection_pool (2.3.0) 42 | coveralls (0.8.23) 43 | json (>= 1.8, < 3) 44 | simplecov (~> 0.16.1) 45 | term-ansicolor (~> 1.3) 46 | thor (>= 0.19.4, < 2.0) 47 | tins (~> 1.6) 48 | crass (1.0.6) 49 | delayed_job (4.1.11) 50 | activesupport (>= 3.0, < 8.0) 51 | diff-lcs (1.5.0) 52 | docile (1.4.0) 53 | e2mmap (0.1.0) 54 | erubi (1.12.0) 55 | fakeredis (0.7.0) 56 | redis (>= 3.2, < 5.0) 57 | globalid (1.0.0) 58 | activesupport (>= 5.0) 59 | i18n (1.12.0) 60 | concurrent-ruby (~> 1.0) 61 | json (2.6.3) 62 | loofah (2.19.1) 63 | crass (~> 1.0.2) 64 | nokogiri (>= 1.5.9) 65 | method_source (1.0.0) 66 | mini_portile2 (2.8.1) 67 | minitest (5.17.0) 68 | mono_logger (1.1.1) 69 | multi_json (1.15.0) 70 | mustermann (3.0.0) 71 | ruby2_keywords (~> 0.0.1) 72 | nokogiri (1.13.10) 73 | mini_portile2 (~> 2.8.0) 74 | racc (~> 1.4) 75 | pry (0.14.1) 76 | coderay (~> 1.1) 77 | method_source (~> 1.0) 78 | pry-byebug (3.10.1) 79 | byebug (~> 11.0) 80 | pry (>= 0.13, < 0.15) 81 | racc (1.6.2) 82 | rack (2.2.5) 83 | rack-protection (3.0.5) 84 | rack 85 | rack-test (2.0.2) 86 | rack (>= 1.3) 87 | rails-dom-testing (2.0.3) 88 | activesupport (>= 4.2.0) 89 | nokogiri (>= 1.6) 90 | rails-html-sanitizer (1.4.4) 91 | loofah (~> 2.19, >= 2.19.1) 92 | railties (5.2.8.1) 93 | actionpack (= 5.2.8.1) 94 | activesupport (= 5.2.8.1) 95 | method_source 96 | rake (>= 0.8.7) 97 | thor (>= 0.19.0, < 2.0) 98 | rake (13.0.6) 99 | redis (4.8.0) 100 | redis-namespace (1.10.0) 101 | redis (>= 4) 102 | resque (2.4.0) 103 | mono_logger (~> 1.0) 104 | multi_json (~> 1.0) 105 | redis-namespace (~> 1.6) 106 | sinatra (>= 0.9.2) 107 | resque_spec (0.18.1) 108 | resque (>= 1.26.0) 109 | rspec-core (>= 3.0.0) 110 | rspec-expectations (>= 3.0.0) 111 | rspec-mocks (>= 3.0.0) 112 | rspec (3.12.0) 113 | rspec-core (~> 3.12.0) 114 | rspec-expectations (~> 3.12.0) 115 | rspec-mocks (~> 3.12.0) 116 | rspec-core (3.12.0) 117 | rspec-support (~> 3.12.0) 118 | rspec-expectations (3.12.1) 119 | diff-lcs (>= 1.2.0, < 2.0) 120 | rspec-support (~> 3.12.0) 121 | rspec-mocks (3.12.1) 122 | diff-lcs (>= 1.2.0, < 2.0) 123 | rspec-support (~> 3.12.0) 124 | rspec-rails (5.1.2) 125 | actionpack (>= 5.2) 126 | activesupport (>= 5.2) 127 | railties (>= 5.2) 128 | rspec-core (~> 3.10) 129 | rspec-expectations (~> 3.10) 130 | rspec-mocks (~> 3.10) 131 | rspec-support (~> 3.10) 132 | rspec-sidekiq (3.1.0) 133 | rspec-core (~> 3.0, >= 3.0.0) 134 | sidekiq (>= 2.4.0) 135 | rspec-support (3.12.0) 136 | ruby2_keywords (0.0.5) 137 | sidekiq (6.5.8) 138 | connection_pool (>= 2.2.5, < 3) 139 | rack (~> 2.0) 140 | redis (>= 4.5.0, < 5) 141 | simplecov (0.16.1) 142 | docile (~> 1.1) 143 | json (>= 1.8, < 3) 144 | simplecov-html (~> 0.10.0) 145 | simplecov-html (0.10.2) 146 | sinatra (3.0.5) 147 | mustermann (~> 3.0) 148 | rack (~> 2.2, >= 2.2.4) 149 | rack-protection (= 3.0.5) 150 | tilt (~> 2.0) 151 | sync (0.5.0) 152 | term-ansicolor (1.7.1) 153 | tins (~> 1.0) 154 | thor (1.2.1) 155 | thread_safe (0.3.6) 156 | thwait (0.2.0) 157 | e2mmap 158 | tilt (2.0.11) 159 | tins (1.32.1) 160 | sync 161 | tzinfo (1.2.10) 162 | thread_safe (~> 0.1) 163 | 164 | PLATFORMS 165 | ruby 166 | 167 | DEPENDENCIES 168 | activejob (>= 5.0.0) 169 | activesupport (~> 5.2.0) 170 | bundler (>= 1.6.0) 171 | coveralls (>= 0.8.22) 172 | delayed_job (~> 4.1.0) 173 | fakeredis (~> 0.7.0) 174 | pry (>= 0.9.0) 175 | pry-byebug (>= 1.3.0) 176 | rake (>= 10.3.0) 177 | resque (>= 1.25.2) 178 | resque_spec (>= 0.16.0) 179 | rspec 180 | rspec-rails (>= 2.0) 181 | rspec-sidekiq (>= 2.1.0) 182 | sidekiq (>= 3.5.0, < 7.0.0) 183 | taskinator! 184 | 185 | BUNDLED WITH 186 | 2.3.5 187 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Chris Stefano 2 | virtualstaticvoid@gmail.com 3 | 4 | MIT License 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining 7 | a copy of this software and associated documentation files (the 8 | "Software"), to deal in the Software without restriction, including 9 | without limitation the rights to use, copy, modify, merge, publish, 10 | distribute, sublicense, and/or sell copies of the Software, and to 11 | permit persons to whom the Software is furnished to do so, subject to 12 | the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be 15 | included in all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | ---------------------------------------------------------------------- 26 | 27 | Portions of code are copied from the Sidekiq project, license below: 28 | 29 | Copyright (c) Mike Perham 30 | https://github.com/mperham/sidekiq 31 | 32 | Sidekiq is an Open Source project licensed under the terms of 33 | the LGPLv3 license. Please see 34 | for license text. 35 | 36 | Sidekiq Pro has a commercial-friendly license allowing private forks 37 | and modifications of Sidekiq. Please see http://sidekiq.org/pro/ for 38 | more detail. You can find the commercial license terms in COMM-LICENSE. 39 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | require 'rspec/core/rake_task' 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task :test => :spec 7 | task :default => :spec 8 | 9 | require 'resque' 10 | require 'resque/tasks' 11 | require 'taskinator' 12 | 13 | Taskinator.configure do |config| 14 | config.logger.level = 0 # DEBUG 15 | config.instrumenter = Taskinator::ConsoleInstrumenter.new 16 | end 17 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "resque" 5 | require "taskinator" 6 | 7 | # You can add fixtures and/or initialization code here to make experimenting 8 | # with your gem easier. You can also use a different console, if you like. 9 | 10 | Taskinator.configure do |config| 11 | config.logger.level = 0 # DEBUG 12 | config.instrumenter = Taskinator::ConsoleInstrumenter.new 13 | end 14 | 15 | require "pry" 16 | Pry.start 17 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | bundle install 6 | 7 | # Do any other automated setup that you need to do here 8 | -------------------------------------------------------------------------------- /lib/taskinator.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | require 'yaml' 3 | require 'securerandom' 4 | require 'benchmark' 5 | require 'delegate' 6 | 7 | require 'taskinator/version' 8 | 9 | require 'taskinator/complete_on' 10 | require 'taskinator/redis_connection' 11 | require 'taskinator/logger' 12 | 13 | require 'taskinator/definition' 14 | 15 | require 'taskinator/workflow' 16 | 17 | require 'taskinator/visitor' 18 | require 'taskinator/persistence' 19 | require 'taskinator/instrumentation' 20 | 21 | require 'taskinator/task' 22 | require 'taskinator/tasks' 23 | require 'taskinator/process' 24 | 25 | require 'taskinator/task_worker' 26 | require 'taskinator/create_process_worker' 27 | 28 | require 'taskinator/executor' 29 | require 'taskinator/queues' 30 | 31 | require 'taskinator/api' 32 | 33 | module Taskinator 34 | 35 | NAME = "Taskinator" 36 | LICENSE = 'See LICENSE.txt for licensing details.' 37 | 38 | DEFAULTS = { 39 | # none for now... 40 | } 41 | 42 | class << self 43 | def options 44 | @options ||= DEFAULTS.dup 45 | end 46 | def options=(opts) 47 | @options = opts 48 | end 49 | 50 | def generate_uuid 51 | SecureRandom.uuid 52 | end 53 | 54 | ## 55 | # Configuration for Taskinator client, use like: 56 | # 57 | # Taskinator.configure do |config| 58 | # config.redis = { :namespace => 'myapp', :pool_size => 1, :url => 'redis://myhost:8877/0' } 59 | # config.queue_config = { :process_queue => 'processes', :task_queue => 'tasks' } 60 | # end 61 | # 62 | def configure 63 | yield self if block_given? 64 | end 65 | 66 | def redis(&block) 67 | raise ArgumentError, "requires a block" unless block_given? 68 | redis_pool.with(&block) 69 | end 70 | 71 | def redis_pool 72 | @redis ||= Taskinator::RedisConnection.create 73 | end 74 | 75 | def redis=(hash) 76 | @redis = Taskinator::RedisConnection.create(hash) 77 | end 78 | 79 | def logger 80 | Taskinator::Logging.logger 81 | end 82 | 83 | def logger=(log) 84 | Taskinator::Logging.logger = log 85 | end 86 | 87 | # the queue adapter to use 88 | # supported adapters include 89 | # :active_job, :delayed_job, :redis and :sidekiq 90 | # NOTE: ensure that the respective gem is included 91 | attr_reader :queue_adapter 92 | 93 | def queue_adapter=(adapter) 94 | @queue_adapter = adapter 95 | @queue = nil 96 | end 97 | 98 | # configuration, usually a hash, which will be passed 99 | # to the configured queue adapter 100 | attr_reader :queue_config 101 | 102 | def queue_config=(config) 103 | @queue_config = config 104 | @queue = nil 105 | end 106 | 107 | def queue 108 | @queue ||= begin 109 | adapter = self.queue_adapter || :resque # TODO: change default to :active_job 110 | config = queue_config || {} 111 | Taskinator::Queues.create_adapter(adapter, config) 112 | end 113 | end 114 | 115 | # set the instrumenter to use. 116 | # can be ActiveSupport::Notifications 117 | def instrumenter 118 | @instrumenter ||= NoOpInstrumenter.new 119 | end 120 | def instrumenter=(value) 121 | @instrumenter = value 122 | end 123 | 124 | end 125 | 126 | class NoOpInstrumenter 127 | def instrument(event, payload={}) 128 | yield(payload) if block_given? 129 | end 130 | end 131 | 132 | class ConsoleInstrumenter 133 | def instrument(event, payload={}) 134 | puts [event.inspect, payload.to_yaml] 135 | yield(payload) if block_given? 136 | end 137 | end 138 | 139 | end 140 | -------------------------------------------------------------------------------- /lib/taskinator/api.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | module Api 3 | class Processes 4 | include Enumerable 5 | 6 | attr_reader :scope 7 | 8 | def initialize(scope=:shared) 9 | @scope = scope 10 | @processes_list_key = Taskinator::Persistence.processes_list_key(scope) 11 | end 12 | 13 | def each(&block) 14 | return to_enum(__method__) unless block_given? 15 | 16 | identifiers = Taskinator.redis do |conn| 17 | conn.smembers(@processes_list_key) 18 | end 19 | 20 | instance_cache = {} 21 | identifiers.each do |identifier| 22 | yield Process.fetch(identifier, instance_cache) 23 | end 24 | end 25 | 26 | def size 27 | Taskinator.redis do |conn| 28 | conn.scard(@processes_list_key) 29 | end 30 | end 31 | end 32 | 33 | def self.find_process(identifier) 34 | Process.fetch(identifier) 35 | end 36 | 37 | def self.find_task(identifier) 38 | Task.fetch(identifier) 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/taskinator/builder.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | class Builder 3 | 4 | attr_reader :process 5 | attr_reader :definition 6 | attr_reader :args 7 | attr_reader :builder_options 8 | 9 | def initialize(process, definition, *args) 10 | @process = process 11 | @definition = definition 12 | @builder_options = args.last.is_a?(Hash) ? args.pop : {} 13 | @args = args 14 | @executor = Taskinator::Executor.new(@definition) 15 | end 16 | 17 | def option?(key, &block) 18 | yield if builder_options[key] 19 | end 20 | 21 | # defines a sub process of tasks which are executed sequentially 22 | def sequential(options={}, &block) 23 | raise ArgumentError, 'block' unless block_given? 24 | 25 | sub_process = Process.define_sequential_process_for(@definition, options) 26 | task = define_sub_process_task(@process, sub_process, options) 27 | Builder.new(sub_process, @definition, *@args).instance_eval(&block) 28 | @process.tasks << task if sub_process.tasks.any? 29 | nil 30 | end 31 | 32 | # defines a sub process of tasks which are executed concurrently 33 | def concurrent(complete_on=CompleteOn::Default, options={}, &block) 34 | raise ArgumentError, 'block' unless block_given? 35 | 36 | sub_process = Process.define_concurrent_process_for(@definition, complete_on, options) 37 | task = define_sub_process_task(@process, sub_process, options) 38 | Builder.new(sub_process, @definition, *@args).instance_eval(&block) 39 | @process.tasks << task if sub_process.tasks.any? 40 | nil 41 | end 42 | 43 | # dynamically defines tasks, using the given @iterator method 44 | # the definition will be evaluated for each yielded item 45 | def for_each(method, options={}, &block) 46 | raise ArgumentError, 'method' if method.nil? 47 | raise NoMethodError, method unless @executor.respond_to?(method) 48 | raise ArgumentError, 'block' unless block_given? 49 | 50 | # 51 | # `for_each` is an exception, since it invokes the definition 52 | # in order to yield elements to the builder, and any options passed 53 | # are included with the builder options 54 | # 55 | method_args = options.any? ? [*@args, options] : @args 56 | @executor.send(method, *method_args) do |*args| 57 | Builder.new(@process, @definition, *args).instance_eval(&block) 58 | end 59 | nil 60 | end 61 | 62 | alias_method :transform, :for_each 63 | 64 | # defines a task which executes the given @method 65 | def task(method, options={}) 66 | raise ArgumentError, 'method' if method.nil? 67 | raise NoMethodError, method unless @executor.respond_to?(method) 68 | 69 | define_step_task(@process, method, @args, options) 70 | nil 71 | end 72 | 73 | # defines a task which executes the given @job 74 | # which is expected to implement a perform method either as a class or instance method 75 | def job(job, options={}) 76 | raise ArgumentError, 'job' if job.nil? 77 | raise ArgumentError, 'job' unless job.methods.include?(:perform) || job.instance_methods.include?(:perform) 78 | 79 | define_job_task(@process, job, @args, options) 80 | nil 81 | end 82 | 83 | # TODO: add mailer 84 | # TODO: add complete! 85 | # TODO: add fail! 86 | 87 | # defines a sub process task, for the given @definition 88 | # the definition specified must have input compatible arguments 89 | # to the current definition 90 | def sub_process(definition, options={}) 91 | raise ArgumentError, 'definition' if definition.nil? 92 | raise ArgumentError, "#{definition.name} does not extend the #{Definition.name} module" unless definition.kind_of?(Definition) 93 | 94 | sub_process = definition.create_sub_process(*@args, combine_options(options)) 95 | task = define_sub_process_task(@process, sub_process, options) 96 | Builder.new(sub_process, definition, *@args) 97 | @process.tasks << task if sub_process.tasks.any? 98 | nil 99 | end 100 | 101 | private 102 | 103 | def define_step_task(process, method, args, options={}) 104 | define_task(process) { 105 | Task.define_step_task(process, method, args, combine_options(options)) 106 | } 107 | end 108 | 109 | def define_job_task(process, job, args, options={}) 110 | define_task(process) { 111 | Task.define_job_task(process, job, args, combine_options(options)) 112 | } 113 | end 114 | 115 | def define_sub_process_task(process, sub_process, options={}) 116 | Task.define_sub_process_task(process, sub_process, combine_options(options)) 117 | end 118 | 119 | def define_task(process) 120 | process.tasks << task = yield 121 | task 122 | end 123 | 124 | def combine_options(options={}) 125 | builder_options.merge(options) 126 | end 127 | 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /lib/taskinator/complete_on.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | module CompleteOn 3 | 4 | # task completion options for concurrent processes 5 | 6 | # completes after the fastest task is completed 7 | # subsequent tasks continue to execute 8 | First = 10 9 | 10 | # completes once all tasks are completed 11 | Last = 20 12 | 13 | # for convenience, the default option 14 | Default = Last 15 | 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/taskinator/create_process_worker.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | class CreateProcessWorker 3 | 4 | attr_reader :definition 5 | attr_reader :uuid 6 | attr_reader :args 7 | 8 | def initialize(definition_name, uuid, args) 9 | 10 | # convert to the module 11 | @definition = constantize(definition_name) 12 | 13 | # this will be uuid of the created process 14 | @uuid = uuid 15 | 16 | # convert to the typed arguments 17 | @args = Taskinator::Persistence.deserialize(args) 18 | 19 | end 20 | 21 | def perform 22 | 23 | # args may contain an options hash at the end 24 | # so merge in the uuid into it, or add 25 | 26 | process_args = args || [] 27 | 28 | if process_args.last.is_a?(Hash) 29 | process_args.last.merge!(:uuid => uuid) 30 | else 31 | process_args << { :uuid => uuid } 32 | end 33 | 34 | @definition._create_process_(false, *process_args).enqueue! 35 | 36 | end 37 | 38 | private 39 | 40 | # :nocov: 41 | def constantize(camel_cased_word) 42 | 43 | # borrowed from activesupport/lib/active_support/inflector/methods.rb 44 | 45 | names = camel_cased_word.split('::') 46 | 47 | # Trigger a built-in NameError exception including the ill-formed constant in the message. 48 | Object.const_get(camel_cased_word) if names.empty? 49 | 50 | # Remove the first blank element in case of '::ClassName' notation. 51 | names.shift if names.size > 1 && names.first.empty? 52 | 53 | names.inject(Object) do |constant, name| 54 | if constant == Object 55 | constant.const_get(name) 56 | else 57 | candidate = constant.const_get(name) 58 | next candidate if constant.const_defined?(name, false) 59 | next candidate unless Object.const_defined?(name) 60 | 61 | # Go down the ancestors to check if it is owned directly. The check 62 | # stops when we reach Object or the end of ancestors tree. 63 | constant = constant.ancestors.inject do |const, ancestor| 64 | break const if ancestor == Object 65 | break ancestor if ancestor.const_defined?(name, false) 66 | const 67 | end 68 | 69 | # owner is in Object, so raise 70 | constant.const_get(name, false) 71 | end 72 | end 73 | end 74 | 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /lib/taskinator/definition.rb: -------------------------------------------------------------------------------- 1 | require 'taskinator/builder' 2 | 3 | module Taskinator 4 | module Definition 5 | 6 | # errors 7 | class ProcessUndefinedError < StandardError; end 8 | class ProcessAlreadyDefinedError < StandardError; end 9 | 10 | # for backward compatibility 11 | UndefinedProcessError = ProcessUndefinedError 12 | 13 | # defines a process 14 | 15 | def define_sequential_process(*arg_list, &block) 16 | factory = lambda {|definition, options| 17 | Process.define_sequential_process_for(definition, options) 18 | } 19 | define_process(*arg_list + [factory], &block) 20 | end 21 | 22 | def define_concurrent_process(*arg_list, &block) 23 | factory = lambda {|definition, options| 24 | complete_on = options.delete(:complete_on) || CompleteOn::Default 25 | Process.define_concurrent_process_for(definition, complete_on, options) 26 | } 27 | define_process(*arg_list + [factory], &block) 28 | end 29 | 30 | def define_process(*arg_list, &block) 31 | raise ProcessAlreadyDefinedError if respond_to?(:_create_process_) 32 | 33 | factory = arg_list.last.respond_to?(:call) ? 34 | arg_list.pop : 35 | lambda {|definition, options| 36 | Process.define_sequential_process_for(definition, options) 37 | } 38 | 39 | # called from respective "create_process" methods 40 | # parameters can contain options as the last parameter 41 | define_singleton_method :_create_process_ do |subprocess, *args| 42 | begin 43 | 44 | # TODO: better validation of arguments 45 | 46 | # FIXME: arg_list should only contain an array of symbols 47 | 48 | raise ArgumentError, "wrong number of arguments (#{args.length} for #{arg_list.length})" if args.length < arg_list.length 49 | 50 | options = (args.last.is_a?(Hash) ? args.last : {}) 51 | options[:scope] ||= :shared 52 | 53 | process = factory.call(self, options) 54 | 55 | # this may take long... up to users definition 56 | Taskinator.instrumenter.instrument('taskinator.process.created', :uuid => process.uuid, :state => :initial) do 57 | Builder.new(process, self, *args).instance_eval(&block) 58 | end 59 | 60 | # only save "root processes" 61 | unless subprocess 62 | 63 | # instrument separately 64 | Taskinator.instrumenter.instrument('taskinator.process.saved', :uuid => process.uuid, :state => :initial) do 65 | 66 | # this will visit "sub processes" and persist them too 67 | process.save 68 | 69 | # add it to the list of "root processes" 70 | Persistence.add_process_to_list(process) 71 | 72 | end 73 | 74 | end 75 | 76 | # this is the "root" process 77 | process 78 | 79 | rescue => e 80 | Taskinator.logger.error(e) 81 | Taskinator.logger.debug(e.backtrace) 82 | raise e 83 | end 84 | end 85 | end 86 | 87 | attr_accessor :queue 88 | 89 | # 90 | # creates an instance of the process 91 | # NOTE: the supplied @args are serialized and ultimately passed to each method of the defined process 92 | # 93 | def create_process(*args) 94 | assert_valid_process_module 95 | _create_process_(false, *args) 96 | end 97 | 98 | # 99 | # returns the process uuid of the process to be created 100 | # the process can be retrieved using this uuid by using 101 | # Taskinator::Process.fetch(uuid) 102 | # 103 | def create_process_remotely(*args) 104 | assert_valid_process_module 105 | uuid = Taskinator.generate_uuid 106 | 107 | Taskinator.queue.enqueue_create_process(self, uuid, args) 108 | 109 | return uuid 110 | end 111 | 112 | def create_sub_process(*args) 113 | assert_valid_process_module 114 | _create_process_(true, *args) 115 | end 116 | 117 | private 118 | 119 | def assert_valid_process_module 120 | raise ProcessUndefinedError unless respond_to?(:_create_process_) 121 | end 122 | 123 | end 124 | end 125 | -------------------------------------------------------------------------------- /lib/taskinator/executor.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | class Executor 3 | 4 | attr_reader :definition 5 | attr_reader :task 6 | 7 | def initialize(definition, task=nil) 8 | @definition = definition 9 | @task = task 10 | 11 | # include the module into the eigen class, so it is only for this instance 12 | eigen = class << self; self; end 13 | eigen.send(:include, definition) 14 | end 15 | 16 | def process_uuid 17 | task.process_uuid if task 18 | end 19 | 20 | def uuid 21 | task.uuid if task 22 | end 23 | 24 | def options 25 | task.options if task 26 | end 27 | 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/taskinator/instrumentation.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | module Instrumentation 3 | 4 | def instrument(event, payload={}) 5 | Taskinator.instrumenter.instrument(event, payload) do 6 | yield 7 | end 8 | end 9 | 10 | # helper methods for instrumentation payloads 11 | 12 | def enqueued_payload(additional={}) 13 | payload_for(:enqueued, additional) 14 | end 15 | 16 | def processing_payload(additional={}) 17 | payload_for(:processing, additional) 18 | end 19 | 20 | def paused_payload(additional={}) 21 | payload_for(:paused, additional) 22 | end 23 | 24 | def resumed_payload(additional={}) 25 | payload_for(:resumed, additional) 26 | end 27 | 28 | def completed_payload(additional={}) 29 | payload_for(:completed, additional) 30 | end 31 | 32 | def cancelled_payload(additional={}) 33 | payload_for(:cancelled, additional) 34 | end 35 | 36 | def failed_payload(exception, additional={}) 37 | payload_for(:failed, { :exception => exception.to_s, :backtrace => exception.backtrace }.merge(additional)) 38 | end 39 | 40 | private 41 | 42 | def payload_for(state, additional={}) 43 | 44 | # need to cache here, since this method hits redis, so can't be part of multi statement following 45 | process_key = self.process_key 46 | 47 | tasks_count, processing_count, completed_count, cancelled_count, failed_count = Taskinator.redis do |conn| 48 | conn.hmget process_key, 49 | :tasks_count, 50 | :tasks_processing, 51 | :tasks_completed, 52 | :tasks_cancelled, 53 | :tasks_failed 54 | end 55 | 56 | tasks_count = tasks_count.to_f 57 | 58 | return OpenStruct.new( 59 | { 60 | :type => self.class.name, 61 | :definition => self.definition.name, 62 | :process_uuid => process_uuid, 63 | :process_options => process_options.dup, 64 | :uuid => uuid, 65 | :options => options.dup, 66 | :state => state, 67 | :percentage_failed => (tasks_count > 0) ? (failed_count.to_i / tasks_count) * 100.0 : 0.0, 68 | :percentage_cancelled => (tasks_count > 0) ? (cancelled_count.to_i / tasks_count) * 100.0 : 0.0, 69 | :percentage_processing => (tasks_count > 0) ? (processing_count.to_i / tasks_count) * 100.0 : 0.0, 70 | :percentage_completed => (tasks_count > 0) ? (completed_count.to_i / tasks_count) * 100.0 : 0.0, 71 | }.merge(additional) 72 | ).freeze 73 | 74 | end 75 | 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /lib/taskinator/logger.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Mike Perham 3 | # 4 | # Sidekiq is an Open Source project licensed under the terms of 5 | # the LGPLv3 license. Please see 6 | # for license text. 7 | # 8 | require 'time' 9 | require 'logger' 10 | 11 | # :nocov: 12 | module Taskinator 13 | module Logging 14 | 15 | class Pretty < Logger::Formatter 16 | # Provide a call() method that returns the formatted message. 17 | def call(severity, time, program_name, message) 18 | "#{time.utc.iso8601} #{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n" 19 | end 20 | 21 | def context 22 | ctx = Thread.current[:taskinator_context] 23 | ctx ? " #{ctx}" : '' 24 | end 25 | end 26 | 27 | class << self 28 | 29 | def initialize_logger(log_target = STDOUT) 30 | oldlogger = defined?(@logger) ? @logger : nil 31 | @logger = Logger.new(log_target) 32 | @logger.level = Logger::INFO 33 | @logger.formatter = Pretty.new 34 | oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging 35 | @logger 36 | end 37 | 38 | def logger 39 | defined?(@logger) ? @logger : initialize_logger 40 | end 41 | 42 | def logger=(log) 43 | @logger = (log ? log : Logger.new('/dev/null')) 44 | end 45 | 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /lib/taskinator/process.rb: -------------------------------------------------------------------------------- 1 | require 'thread' 2 | require 'thwait' 3 | 4 | module Taskinator 5 | class Process 6 | include ::Comparable 7 | 8 | include Workflow 9 | include Persistence 10 | include Instrumentation 11 | 12 | class << self 13 | def define_sequential_process_for(definition, options={}) 14 | Process::Sequential.new(definition, options) 15 | end 16 | 17 | def define_concurrent_process_for(definition, complete_on=CompleteOn::Default, options={}) 18 | Process::Concurrent.new(definition, complete_on, options) 19 | end 20 | end 21 | 22 | attr_reader :uuid 23 | attr_reader :definition 24 | attr_reader :options 25 | attr_reader :scope 26 | attr_reader :queue 27 | attr_reader :created_at 28 | attr_reader :updated_at 29 | 30 | # in the case of sub process tasks, the containing task 31 | attr_reader :parent 32 | 33 | def initialize(definition, options={}) 34 | raise ArgumentError, 'definition' if definition.nil? 35 | raise ArgumentError, "#{definition.name} does not extend the #{Definition.name} module" unless definition.kind_of?(Definition) 36 | 37 | @uuid = options.delete(:uuid) || Taskinator.generate_uuid 38 | @definition = definition 39 | @options = options 40 | @scope = options.delete(:scope) 41 | @queue = options.delete(:queue) 42 | @created_at = Time.now.utc 43 | @updated_at = created_at 44 | @current_state = :initial 45 | end 46 | 47 | def parent=(value) 48 | @parent = value 49 | # update the uuid to be "scoped" within the parent task 50 | @uuid = "#{@parent.uuid}:subprocess" 51 | @key = nil # NB: invalidate memoized key 52 | end 53 | 54 | def tasks 55 | @tasks ||= Tasks.new 56 | end 57 | 58 | def no_tasks_defined? 59 | tasks.empty? 60 | end 61 | 62 | def accept(visitor) 63 | visitor.visit_attribute(:uuid) 64 | visitor.visit_task_reference(:parent) 65 | visitor.visit_type(:definition) 66 | visitor.visit_tasks(tasks) 67 | visitor.visit_args(:options) 68 | visitor.visit_attribute(:scope) 69 | visitor.visit_attribute(:queue) 70 | visitor.visit_attribute_time(:created_at) 71 | visitor.visit_attribute_time(:updated_at) 72 | end 73 | 74 | def <=>(other) 75 | uuid <=> other.uuid 76 | end 77 | 78 | def to_s 79 | "#<#{self.class.name}:#{uuid}>" 80 | end 81 | 82 | def enqueue! 83 | return if paused? || cancelled? 84 | 85 | transition(:enqueued) do 86 | instrument('taskinator.process.enqueued', enqueued_payload) do 87 | enqueue 88 | end 89 | end 90 | end 91 | 92 | def start! 93 | return if paused? || cancelled? 94 | 95 | transition(:processing) do 96 | instrument('taskinator.process.processing', processing_payload) do 97 | start 98 | end 99 | end 100 | end 101 | 102 | def pause! 103 | return unless enqueued? || processing? 104 | 105 | transition(:paused) do 106 | instrument('taskinator.process.paused', paused_payload) do 107 | pause if respond_to?(:pause) 108 | end 109 | end 110 | end 111 | 112 | def resume! 113 | return unless paused? 114 | 115 | transition(:processing) do 116 | instrument('taskinator.process.resumed', resumed_payload) do 117 | resume if respond_to?(:resume) 118 | end 119 | end 120 | end 121 | 122 | def complete! 123 | transition(:completed) do 124 | instrument('taskinator.process.completed', completed_payload) do 125 | complete if respond_to?(:complete) 126 | # notify the parent task (if there is one) that this process has completed 127 | # note: parent may be a proxy, so explicitly check for nil? 128 | unless parent.nil? 129 | parent.complete! 130 | else 131 | cleanup 132 | end 133 | end 134 | end 135 | end 136 | 137 | # TODO: add retry method - to pick up from a failed task 138 | # e.g. like retrying a failed job in Resque Web 139 | 140 | def tasks_completed? 141 | # TODO: optimize this 142 | tasks.all?(&:completed?) 143 | end 144 | 145 | def cancel! 146 | transition(:cancelled) do 147 | instrument('taskinator.process.cancelled', cancelled_payload) do 148 | cancel if respond_to?(:cancel) 149 | end 150 | end 151 | end 152 | 153 | def fail!(error) 154 | transition(:failed) do 155 | instrument('taskinator.process.failed', failed_payload(error)) do 156 | fail(error) if respond_to?(:fail) 157 | # notify the parent task (if there is one) that this process has failed 158 | # note: parent may be a proxy, so explicitly check for nil? 159 | parent.fail!(error) unless parent.nil? 160 | end 161 | end 162 | end 163 | 164 | def task_failed(task, error) 165 | # for now, fail this process 166 | fail!(error) 167 | end 168 | 169 | #-------------------------------------------------- 170 | # subclasses must implement the following methods 171 | #-------------------------------------------------- 172 | 173 | # :nocov: 174 | def enqueue 175 | raise NotImplementedError 176 | end 177 | 178 | def start 179 | raise NotImplementedError 180 | end 181 | 182 | def task_completed(task) 183 | raise NotImplementedError 184 | end 185 | # :nocov: 186 | 187 | #-------------------------------------------------- 188 | 189 | class Sequential < Process 190 | def enqueue 191 | if tasks.empty? 192 | complete! # weren't any tasks to start with 193 | else 194 | tasks.first.enqueue! 195 | end 196 | end 197 | 198 | def start 199 | task = tasks.first 200 | if task 201 | task.start! 202 | else 203 | complete! # weren't any tasks to start with 204 | end 205 | end 206 | 207 | def task_completed(task) 208 | # decrement the count of pending sequential tasks 209 | pending = deincr_pending_tasks 210 | 211 | Taskinator.logger.info("Completed task for process '#{uuid}'. Pending is #{pending}.") 212 | 213 | next_task = task.next 214 | if next_task 215 | next_task.enqueue! 216 | else 217 | complete! # aren't any more tasks 218 | end 219 | end 220 | 221 | def inspect 222 | %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", state=:#{current_state}, tasks=[#{tasks.inspect}]>) 223 | end 224 | end 225 | 226 | #-------------------------------------------------- 227 | 228 | class Concurrent < Process 229 | attr_reader :complete_on 230 | 231 | # DEPRECATED: concurrency_method will be removed in a future version. 232 | attr_reader :concurrency_method 233 | 234 | def initialize(definition, complete_on=CompleteOn::Default, options={}) 235 | super(definition, options) 236 | @complete_on = complete_on 237 | @concurrency_method = options.delete(:concurrency_method) || :thread 238 | warn("[DEPRECATED]: concurrency_method will be removed in a future version.") if @concurrency_method == :fork 239 | end 240 | 241 | def enqueue 242 | if tasks.empty? 243 | complete! # weren't any tasks to start with 244 | else 245 | Taskinator.logger.info("Enqueuing #{tasks.count} tasks for process '#{uuid}'.") 246 | tasks.each(&:enqueue!) 247 | end 248 | end 249 | 250 | # this method only called in-process (usually from the console) 251 | def start 252 | if tasks.empty? 253 | complete! # weren't any tasks to start with 254 | else 255 | if concurrency_method == :fork 256 | # :nocov: 257 | warn("[DEPRECATED]: concurrency_method will be removed in a future version.") 258 | tasks.each do |task| 259 | fork do 260 | task.start! 261 | end 262 | end 263 | Process.waitall 264 | # :nocov: 265 | else 266 | threads = tasks.map do |task| 267 | Thread.new do 268 | task.start! 269 | end 270 | end 271 | ThreadsWait.all_waits(*threads) 272 | end 273 | end 274 | end 275 | 276 | def task_completed(task) 277 | # decrement the count of pending concurrent tasks 278 | pending = deincr_pending_tasks 279 | 280 | Taskinator.logger.info("Completed task for process '#{uuid}'. Pending is #{pending}.") 281 | 282 | # when complete on first, then don't bother with subsequent tasks completing 283 | if complete_on == CompleteOn::First 284 | complete! unless completed? 285 | else 286 | complete! if pending < 1 287 | end 288 | end 289 | 290 | def tasks_completed? 291 | if complete_on == CompleteOn::First 292 | tasks.any?(&:completed?) 293 | else 294 | super # all 295 | end 296 | end 297 | 298 | def accept(visitor) 299 | super 300 | visitor.visit_attribute_enum(:complete_on, CompleteOn) 301 | end 302 | 303 | def inspect 304 | %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", state=:#{current_state}, complete_on=:#{complete_on}, tasks=[#{tasks.inspect}]>) 305 | end 306 | end 307 | end 308 | end 309 | -------------------------------------------------------------------------------- /lib/taskinator/queues.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | module Queues 3 | 4 | DefaultConfig = { 5 | :definition_queue => :default, 6 | :process_queue => :default, 7 | :task_queue => :default 8 | }.freeze 9 | 10 | def self.create_adapter(adapter, config={}) 11 | begin 12 | LoggedAdapter.new(send("create_#{adapter}_adapter", config)) 13 | rescue NoMethodError 14 | raise "The queue adapter `#{adapter}` is not yet supported or it's runtime isn't loaded." 15 | end 16 | end 17 | 18 | class LoggedAdapter < Delegator 19 | 20 | attr_reader :adapter 21 | 22 | def initialize(adapter) 23 | Taskinator.logger.info("Initialized '#{adapter.class.name}' queue adapter") 24 | @adapter = adapter 25 | end 26 | 27 | def __getobj__ 28 | adapter 29 | end 30 | 31 | def enqueue_create_process(definition, uuid, args) 32 | Taskinator.logger.info("Enqueuing process creation for #{definition}") 33 | adapter.enqueue_create_process(definition, uuid, args) 34 | end 35 | 36 | def enqueue_task(task) 37 | Taskinator.logger.info("Enqueuing task #{task}") 38 | adapter.enqueue_task(task) 39 | end 40 | 41 | end 42 | 43 | end 44 | end 45 | 46 | require 'taskinator/queues/active_job' if defined?(ApplicationJob) 47 | require 'taskinator/queues/delayed_job' if defined?(Delayed) 48 | require 'taskinator/queues/resque' if defined?(Resque) 49 | require 'taskinator/queues/sidekiq' if defined?(Sidekiq) 50 | -------------------------------------------------------------------------------- /lib/taskinator/queues/active_job.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | module Queues 3 | 4 | # https://guides.rubyonrails.org/active_job_basics.html 5 | 6 | def self.create_active_job_adapter(config={}) 7 | ActiveJobAdapter.new(config) 8 | end 9 | 10 | class ActiveJobAdapter 11 | def initialize(config={}) 12 | @config = Taskinator::Queues::DefaultConfig.merge(config) 13 | end 14 | 15 | def enqueue_create_process(definition, uuid, args) 16 | queue = definition.queue || @config[:definition_queue] 17 | CreateProcessWorker.set(:queue => queue) 18 | .perform_later(definition.name, uuid, Taskinator::Persistence.serialize(args)) 19 | end 20 | 21 | def enqueue_task(task) 22 | queue = task.queue || @config[:task_queue] 23 | TaskWorker.set(:queue => queue) 24 | .perform_later(task.uuid) 25 | end 26 | 27 | class CreateProcessWorker < ApplicationJob 28 | def perform(definition_name, uuid, args) 29 | Taskinator::CreateProcessWorker.new(definition_name, uuid, args).perform 30 | end 31 | end 32 | 33 | class TaskWorker < ApplicationJob 34 | def perform(task_uuid) 35 | Taskinator::TaskWorker.new(task_uuid).perform 36 | end 37 | end 38 | 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/taskinator/queues/delayed_job.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | module Queues 3 | 4 | # https://github.com/collectiveidea/delayed_job 5 | 6 | def self.create_delayed_job_adapter(config={}) 7 | DelayedJobAdapter.new(config) 8 | end 9 | 10 | class DelayedJobAdapter 11 | def initialize(config={}) 12 | @config = Taskinator::Queues::DefaultConfig.merge(config) 13 | end 14 | 15 | def enqueue_create_process(definition, uuid, args) 16 | queue = definition.queue || @config[:definition_queue] 17 | ::Delayed::Job.enqueue CreateProcessWorker.new(definition.name, uuid, Taskinator::Persistence.serialize(args)), :queue => queue 18 | end 19 | 20 | def enqueue_task(task) 21 | queue = task.queue || @config[:task_queue] 22 | ::Delayed::Job.enqueue TaskWorker.new(task.uuid), :queue => queue 23 | end 24 | 25 | CreateProcessWorker = Struct.new(:definition_name, :uuid, :args) do 26 | def perform 27 | Taskinator::CreateProcessWorker.new(definition_name, uuid, args).perform 28 | end 29 | end 30 | 31 | TaskWorker = Struct.new(:task_uuid) do 32 | def perform 33 | Taskinator::TaskWorker.new(task_uuid).perform 34 | end 35 | end 36 | 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/taskinator/queues/resque.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | module Queues 3 | 4 | # https://github.com/resque/resque 5 | 6 | def self.create_resque_adapter(config={}) 7 | ResqueAdapter.new(config) 8 | end 9 | 10 | class ResqueAdapter 11 | def initialize(config={}) 12 | config = Taskinator::Queues::DefaultConfig.merge(config) 13 | 14 | CreateProcessWorker.class_eval do 15 | @queue = config[:definition_queue] 16 | end 17 | 18 | TaskWorker.class_eval do 19 | @queue = config[:task_queue] 20 | end 21 | 22 | end 23 | 24 | def enqueue_create_process(definition, uuid, args) 25 | queue = definition.queue || Resque.queue_from_class(CreateProcessWorker) 26 | Resque.enqueue_to(queue, CreateProcessWorker, definition.name, uuid, Taskinator::Persistence.serialize(args)) 27 | end 28 | 29 | def enqueue_task(task) 30 | queue = task.queue || Resque.queue_from_class(TaskWorker) 31 | Resque.enqueue_to(queue, TaskWorker, task.uuid) 32 | end 33 | 34 | class CreateProcessWorker 35 | def self.perform(definition_name, uuid, args) 36 | Taskinator::CreateProcessWorker.new(definition_name, uuid, args).perform 37 | end 38 | end 39 | 40 | class TaskWorker 41 | def self.perform(task_uuid) 42 | Taskinator::TaskWorker.new(task_uuid).perform 43 | end 44 | end 45 | 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /lib/taskinator/queues/sidekiq.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | module Queues 3 | 4 | # https://github.com/mperham/sidekiq 5 | 6 | def self.create_sidekiq_adapter(config={}) 7 | SidekiqAdapter.new(config) 8 | end 9 | 10 | class SidekiqAdapter 11 | def initialize(config={}) 12 | @config = Taskinator::Queues::DefaultConfig.merge(config) 13 | end 14 | 15 | def enqueue_create_process(definition, uuid, args) 16 | queue = definition.queue || @config[:definition_queue] 17 | CreateProcessWorker.client_push('class' => CreateProcessWorker, 'args' => [definition.name, uuid, Taskinator::Persistence.serialize(args)], 'queue' => queue) 18 | end 19 | 20 | def enqueue_task(task) 21 | queue = task.queue || @config[:task_queue] 22 | TaskWorker.client_push('class' => TaskWorker, 'args' => [task.uuid], 'queue' => queue) 23 | end 24 | 25 | class CreateProcessWorker 26 | include ::Sidekiq::Worker 27 | 28 | def perform(definition_name, uuid, args) 29 | Taskinator::CreateProcessWorker.new(definition_name, uuid, args).perform 30 | end 31 | end 32 | 33 | class TaskWorker 34 | include ::Sidekiq::Worker 35 | 36 | def perform(task_uuid) 37 | Taskinator::TaskWorker.new(task_uuid).perform 38 | end 39 | end 40 | 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/taskinator/redis_connection.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Mike Perham 3 | # 4 | # Sidekiq is an Open Source project licensed under the terms of 5 | # the LGPLv3 license. Please see 6 | # for license text. 7 | # 8 | # Sidekiq Pro has a commercial-friendly license allowing private forks 9 | # and modifications of Sidekiq. Please see http://sidekiq.org/pro/ for 10 | # more detail. You can find the commercial license terms in COMM-LICENSE. 11 | # 12 | 13 | require 'connection_pool' 14 | require 'redis' 15 | require 'uri' 16 | 17 | # :nocov: 18 | module Taskinator 19 | class RedisConnection 20 | class << self 21 | 22 | def create(options={}) 23 | url = options[:url] || determine_redis_provider 24 | if url 25 | options[:url] = url 26 | end 27 | 28 | pool_size = options[:pool_size] || 5 29 | pool_timeout = options[:pool_timeout] || 1 30 | 31 | log_info(options) 32 | 33 | ConnectionPool.new(:timeout => pool_timeout, :size => pool_size) do 34 | build_client(options) 35 | end 36 | end 37 | 38 | private 39 | 40 | def build_client(options) 41 | namespace = options[:namespace] 42 | 43 | client = Redis.new client_opts(options) 44 | if namespace 45 | require 'redis/namespace' 46 | Redis::Namespace.new(namespace, :redis => client) 47 | else 48 | client 49 | end 50 | end 51 | 52 | def client_opts(options) 53 | opts = options.dup 54 | if opts[:namespace] 55 | opts.delete(:namespace) 56 | end 57 | 58 | if opts[:network_timeout] 59 | opts[:timeout] = opts[:network_timeout] 60 | opts.delete(:network_timeout) 61 | end 62 | 63 | opts 64 | end 65 | 66 | def log_info(options) 67 | # Don't log Redis AUTH password 68 | scrubbed_options = options.dup 69 | if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password 70 | uri.password = "REDACTED" 71 | scrubbed_options[:url] = uri.to_s 72 | end 73 | Taskinator.logger.info("#{Taskinator::NAME} client with redis options #{scrubbed_options}") 74 | end 75 | 76 | def determine_redis_provider 77 | ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL'] 78 | end 79 | 80 | end 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /lib/taskinator/task.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | class Task 3 | include ::Comparable 4 | 5 | include Workflow 6 | include Persistence 7 | include Instrumentation 8 | 9 | class << self 10 | def define_step_task(process, method, args, options={}) 11 | Step.new(process, method, args, options) 12 | end 13 | 14 | def define_job_task(process, job, args, options={}) 15 | Job.new(process, job, args, options) 16 | end 17 | 18 | def define_sub_process_task(process, sub_process, options={}) 19 | SubProcess.new(process, sub_process, options) 20 | end 21 | end 22 | 23 | attr_reader :process 24 | attr_reader :definition 25 | attr_reader :uuid 26 | attr_reader :options 27 | attr_reader :queue 28 | attr_reader :created_at 29 | attr_reader :updated_at 30 | 31 | # the next task in the sequence 32 | attr_accessor :next 33 | 34 | def initialize(process, options={}) 35 | raise ArgumentError, 'process' if process.nil? || !process.is_a?(Process) 36 | 37 | @uuid = "#{process.uuid}:task:#{Taskinator.generate_uuid}" 38 | @process = process 39 | @definition = process.definition 40 | @options = options 41 | @queue = options.delete(:queue) 42 | @created_at = Time.now.utc 43 | @updated_at = created_at 44 | @current_state = :initial 45 | end 46 | 47 | def accept(visitor) 48 | visitor.visit_attribute(:uuid) 49 | visitor.visit_process_reference(:process) 50 | visitor.visit_type(:definition) 51 | visitor.visit_task_reference(:next) 52 | visitor.visit_args(:options) 53 | visitor.visit_attribute(:queue) 54 | visitor.visit_attribute_time(:created_at) 55 | visitor.visit_attribute_time(:updated_at) 56 | end 57 | 58 | def <=>(other) 59 | uuid <=> other.uuid 60 | end 61 | 62 | def to_s 63 | "#<#{self.class.name}:#{uuid}>" 64 | end 65 | 66 | def enqueue! 67 | return if paused? || cancelled? 68 | 69 | transition(:enqueued) do 70 | instrument('taskinator.task.enqueued', enqueued_payload) do 71 | enqueue 72 | end 73 | end 74 | end 75 | 76 | def start! 77 | return if paused? || cancelled? 78 | self.incr_processing if incr_count? 79 | 80 | transition(:processing) do 81 | instrument('taskinator.task.processing', processing_payload) do 82 | start 83 | end 84 | end 85 | end 86 | 87 | # 88 | # NOTE: a task can't be paused (it's too difficult to implement) 89 | # so rather, the parent process is paused, and the task checks it 90 | # 91 | 92 | # helper method 93 | def paused? 94 | super || process.paused? 95 | end 96 | 97 | def complete! 98 | transition(:completed) do 99 | self.incr_completed if incr_count? 100 | instrument('taskinator.task.completed', completed_payload) do 101 | complete if respond_to?(:complete) 102 | # notify the process that this task has completed 103 | process.task_completed(self) 104 | end 105 | end 106 | end 107 | 108 | def cancel! 109 | transition(:cancelled) do 110 | self.incr_cancelled if incr_count? 111 | instrument('taskinator.task.cancelled', cancelled_payload) do 112 | cancel if respond_to?(:cancel) 113 | end 114 | end 115 | end 116 | 117 | def cancelled? 118 | super || process.cancelled? 119 | end 120 | 121 | def fail!(error) 122 | transition(:failed) do 123 | self.incr_failed if incr_count? 124 | instrument('taskinator.task.failed', failed_payload(error)) do 125 | fail(error) if respond_to?(:fail) 126 | # notify the process that this task has failed 127 | process.task_failed(self, error) 128 | end 129 | end 130 | end 131 | 132 | def incr_count? 133 | true 134 | end 135 | 136 | #-------------------------------------------------- 137 | # subclasses must implement the following methods 138 | #-------------------------------------------------- 139 | 140 | # :nocov: 141 | def enqueue 142 | raise NotImplementedError 143 | end 144 | 145 | def start 146 | raise NotImplementedError 147 | end 148 | # :nocov: 149 | 150 | #-------------------------------------------------- 151 | # and optionally, provide methods: 152 | #-------------------------------------------------- 153 | # 154 | # * cancel 155 | # * complete 156 | # * fail(error) 157 | # 158 | #-------------------------------------------------- 159 | 160 | # a task which invokes the specified method on the definition 161 | # the args must be intrinsic types, since they are serialized to YAML 162 | class Step < Task 163 | attr_reader :method 164 | attr_reader :args 165 | 166 | def initialize(process, method, args, options={}) 167 | super(process, options) 168 | 169 | raise ArgumentError, 'method' if method.nil? 170 | raise NoMethodError, method unless executor.respond_to?(method) 171 | 172 | @method = method 173 | @args = args 174 | end 175 | 176 | def enqueue 177 | Taskinator.queue.enqueue_task(self) 178 | end 179 | 180 | def start 181 | executor.send(method, *args) 182 | # ASSUMPTION: when the method returns, the task is considered to be complete 183 | complete! 184 | 185 | rescue => e 186 | Taskinator.logger.error(e) 187 | Taskinator.logger.debug(e.backtrace) 188 | fail!(e) 189 | raise e 190 | end 191 | 192 | def accept(visitor) 193 | super 194 | visitor.visit_attribute(:method) 195 | visitor.visit_args(:args) 196 | end 197 | 198 | def executor 199 | @executor ||= Taskinator::Executor.new(definition, self) 200 | end 201 | 202 | def inspect 203 | %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", definition=:#{definition}, method=:#{method}, args=#{args}, current_state=:#{current_state}>) 204 | end 205 | end 206 | 207 | #-------------------------------------------------- 208 | 209 | # a task which invokes the specified background job 210 | # the args must be intrinsic types, since they are serialized to YAML 211 | class Job < Task 212 | attr_reader :job 213 | attr_reader :args 214 | 215 | def initialize(process, job, args, options={}) 216 | super(process, options) 217 | 218 | raise ArgumentError, 'job' if job.nil? 219 | raise ArgumentError, 'job' unless job.methods.include?(:perform) || job.instance_methods.include?(:perform) 220 | 221 | @job = job 222 | @args = args 223 | end 224 | 225 | def enqueue 226 | Taskinator.queue.enqueue_task(self) 227 | end 228 | 229 | def start 230 | # NNB: if other job types are required, may need to implement how they get invoked here! 231 | 232 | if job.respond_to?(:perform) 233 | # resque 234 | job.perform(*args) 235 | else 236 | # delayedjob and sidekiq 237 | job.new.perform(*args) 238 | end 239 | 240 | # ASSUMPTION: when the job returns, the task is considered to be complete 241 | complete! 242 | 243 | rescue => e 244 | Taskinator.logger.error(e) 245 | Taskinator.logger.debug(e.backtrace) 246 | fail!(e) 247 | raise e 248 | end 249 | 250 | def accept(visitor) 251 | super 252 | visitor.visit_type(:job) 253 | visitor.visit_args(:args) 254 | end 255 | 256 | def inspect 257 | %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", definition=:#{definition}, job=#{job}, args=#{args}, current_state=:#{current_state}>) 258 | end 259 | end 260 | 261 | #-------------------------------------------------- 262 | 263 | # a task which delegates to another process 264 | class SubProcess < Task 265 | attr_reader :sub_process 266 | 267 | # NOTE: also wraps sequential and concurrent processes 268 | 269 | def initialize(process, sub_process, options={}) 270 | super(process, options) 271 | raise ArgumentError, 'sub_process' if sub_process.nil? || !sub_process.is_a?(Process) 272 | 273 | @sub_process = sub_process 274 | @sub_process.parent = self 275 | end 276 | 277 | def enqueue 278 | sub_process.enqueue! 279 | end 280 | 281 | def start 282 | sub_process.start! 283 | 284 | rescue => e 285 | Taskinator.logger.error(e) 286 | Taskinator.logger.debug(e.backtrace) 287 | fail!(e) 288 | raise e 289 | end 290 | 291 | def incr_count? 292 | # subprocess tasks aren't included in the total count of tasks 293 | # since they simply delegate to the tasks of the respective subprocess definition 294 | false 295 | end 296 | 297 | def accept(visitor) 298 | super 299 | visitor.visit_process(:sub_process) 300 | end 301 | 302 | def inspect 303 | %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", definition=:#{definition}, sub_process=#{sub_process.inspect}, current_state=:#{current_state}>) 304 | end 305 | end 306 | end 307 | end 308 | -------------------------------------------------------------------------------- /lib/taskinator/task_worker.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | class TaskWorker 3 | attr_reader :uuid 4 | 5 | def initialize(uuid) 6 | @uuid = uuid 7 | end 8 | 9 | def perform 10 | task = Taskinator::Task.fetch(@uuid) 11 | raise "ERROR: Task '#{@uuid}' not found." unless task 12 | return if task.paused? || task.cancelled? 13 | task.start! 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/taskinator/tasks.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | class Tasks 3 | include Enumerable 4 | 5 | # implements a linked list, where each task references the next task 6 | 7 | attr_reader :head 8 | alias_method :first, :head 9 | 10 | attr_reader :count 11 | alias_method :length, :count 12 | 13 | def initialize(first=nil) 14 | @count = 0 15 | add(first) if first 16 | end 17 | 18 | def attach(task, count) 19 | @head = task 20 | @count = count 21 | task 22 | end 23 | 24 | def add(task) 25 | if @head.nil? 26 | @head = task 27 | @count = 1 28 | else 29 | current = @head 30 | while current.next 31 | current = current.next 32 | end 33 | current.next = task 34 | @count += 1 35 | end 36 | task 37 | end 38 | 39 | alias_method :<<, :add 40 | alias_method :push, :add 41 | 42 | def empty? 43 | @head.nil? 44 | end 45 | 46 | def each(&block) 47 | return to_enum(__method__) unless block_given? 48 | 49 | current = @head 50 | while current 51 | yield current 52 | current = current.next 53 | end 54 | end 55 | 56 | def inspect 57 | %([#{collect(&:inspect).join(', ')}]) 58 | end 59 | 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/taskinator/version.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | VERSION = "0.5.2" 3 | end 4 | -------------------------------------------------------------------------------- /lib/taskinator/visitor.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | module Visitor 3 | class Base 4 | def visit_process(attribute) 5 | end 6 | 7 | def visit_tasks(tasks) 8 | end 9 | 10 | def visit_attribute(attribute) 11 | end 12 | 13 | def visit_attribute_time(attribute) 14 | end 15 | 16 | def visit_attribute_enum(attribute, type) 17 | end 18 | 19 | def visit_process_reference(attribute) 20 | end 21 | 22 | def visit_task_reference(attribute) 23 | end 24 | 25 | def visit_type(attribute) 26 | end 27 | 28 | def visit_args(attribute) 29 | end 30 | 31 | def task_count 32 | # return the total count of all tasks 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/taskinator/workflow.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | module Workflow 3 | 4 | def current_state 5 | @current_state ||= load_workflow_state 6 | end 7 | 8 | def current_state=(new_state) 9 | return if new_state == @current_state 10 | @current_state = persist_workflow_state(new_state) 11 | end 12 | 13 | def transition(new_state) 14 | self.current_state = new_state 15 | yield if block_given? 16 | current_state 17 | end 18 | 19 | %i( 20 | initial 21 | enqueued 22 | processing 23 | paused 24 | resumed 25 | completed 26 | cancelled 27 | failed 28 | ).each do |state| 29 | 30 | define_method :"#{state}?" do 31 | current_state == state 32 | end 33 | 34 | end 35 | 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /processes_workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/virtualstaticvoid/taskinator/cc8408de02dc931c3d699888f20ac3be2aaacf27/processes_workflow.png -------------------------------------------------------------------------------- /sequence.txt: -------------------------------------------------------------------------------- 1 | # https://www.websequencediagrams.com 2 | title Taskinator Sequence 3 | 4 | User->+Web: Request process 5 | Web-->+Process(seq): Create sequential process 6 | Process(seq)-->-Web: 7 | Web-->+Queue: Enqueue process 8 | Queue-->-Web: 9 | Web-->-User: 10 | 11 | opt Sequential process 12 | Queue->+Worker: Dequeue process 13 | note right of Worker: Start sequential process 14 | Worker->+Queue: Enqueue task 15 | Queue-->-Worker: 16 | Worker-->-Queue: 17 | 18 | loop Sequential Tasks 19 | Queue->+Worker: Dequeue task 20 | note right of Worker: Start task 21 | Worker->+Queue: Enqueue task 22 | Queue-->-Worker: 23 | Worker->Process(seq): Task completed 24 | note left of Process(seq): All tasks complete? 25 | Worker-->-Queue: 26 | end 27 | 28 | opt Sub Process Task 29 | Queue->+Worker: Dequeue task 30 | note right of Worker: Start task 31 | Worker-->+Process(con): Create concurrent process 32 | Process(con)-->-Worker: 33 | Worker-->+Queue: Enqueue process 34 | Queue-->-Worker: 35 | Worker->Process(seq): Task completed 36 | note left of Process(seq): All tasks complete? 37 | Worker-->-Queue: 38 | end 39 | 40 | opt Concurrent process 41 | Queue->+Worker: Dequeue process 42 | note right of Worker: Start concurrent process 43 | Worker->+Queue: Enqueue task 44 | Queue-->-Worker: 45 | Worker->+Queue: Enqueue task 46 | Queue-->-Worker: 47 | Worker->Process(seq): Task completed 48 | note left of Process(seq): All tasks complete? 49 | Worker-->-Queue: 50 | 51 | opt Concurrent Tasks 52 | Queue->+Worker: Dequeue task 53 | Queue->+Worker: Dequeue task 54 | Queue->+Worker: Dequeue task 55 | Worker->Process(con): Task completed 56 | note right of Process(con): All tasks complete? 57 | Worker->Process(con): Task completed 58 | note right of Process(con): All tasks complete? 59 | Worker-->-Queue: 60 | Worker-->-Queue: 61 | Worker->Process(con): Task completed 62 | Worker-->-Queue: 63 | end 64 | 65 | note right of Process(con): All tasks complete? 66 | Process(con)->Process(seq): Process completed 67 | note left of Process(seq): All tasks complete? 68 | 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /spec/examples/process_examples.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | shared_examples_for "a process" do |process_type| 4 | 5 | # NOTE: definition and subject must be defined by callee 6 | 7 | it { expect(subject.definition).to eq(definition) } 8 | it { expect(subject.uuid).to_not be_nil } 9 | it { expect(subject.to_s).to match(/#{subject.uuid}/) } 10 | it { expect(subject.options).to_not be_nil } 11 | it { expect(subject.tasks).to_not be_nil } 12 | 13 | end 14 | -------------------------------------------------------------------------------- /spec/examples/queue_adapter_examples.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | shared_examples_for "a queue adapter" do |adapter_name, adapter_type| 4 | 5 | subject { adapter_type.new({}) } 6 | let(:job) { double('job') } 7 | 8 | it "should instantiate adapter" do 9 | Taskinator.queue_adapter = adapter_name 10 | expect(Taskinator.queue.adapter).to be_a(adapter_type) 11 | end 12 | 13 | describe "#enqueue_create_process" do 14 | it { expect(subject).to respond_to(:enqueue_create_process) } 15 | 16 | it "should enqueue a create process" do 17 | expect( 18 | subject.enqueue_create_process(double('definition', :name => 'definition', :queue => nil), 'xx-xx-xx-xx', :foo => :bar) 19 | ).to_not be_nil 20 | end 21 | end 22 | 23 | describe "#enqueue_task" do 24 | it { expect(subject).to respond_to(:enqueue_task) } 25 | 26 | it "should enqueue a task" do 27 | expect( 28 | subject.enqueue_task(double('task', :uuid => 'xx-xx-xx-xx', :queue => nil)) 29 | ).to_not be_nil 30 | end 31 | end 32 | 33 | end 34 | -------------------------------------------------------------------------------- /spec/examples/task_examples.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | shared_examples_for "a task" do |task_type| 4 | 5 | # NOTE: process and subject must be defined by callee 6 | 7 | it { expect(subject.process).to eq(process) } 8 | it { expect(subject.uuid).to_not be_nil } 9 | it { expect(subject.to_s).to match(/#{subject.uuid}/) } 10 | it { expect(subject.options).to_not be_nil } 11 | 12 | end 13 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | Bundler.setup 3 | 4 | require 'simplecov' 5 | require 'coveralls' 6 | require 'pry' 7 | require 'active_support/notifications' 8 | 9 | SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([ 10 | SimpleCov::Formatter::HTMLFormatter, 11 | Coveralls::SimpleCov::Formatter 12 | ]) 13 | 14 | SimpleCov.start do 15 | add_filter 'spec' 16 | end 17 | 18 | require 'fakeredis/rspec' 19 | 20 | require 'delayed_job' 21 | 22 | require 'sidekiq' 23 | require 'rspec-sidekiq' 24 | Sidekiq::Testing.fake! 25 | 26 | require 'resque' 27 | require 'resque_spec' 28 | ResqueSpec.disable_ext = false 29 | 30 | require 'active_job' 31 | 32 | ActiveJob::Base.queue_adapter = :test 33 | 34 | class ApplicationJob < ActiveJob::Base 35 | queue_as :not_used 36 | end 37 | 38 | # minimum rails gems for rspec/rails 39 | require 'action_view' 40 | require 'action_dispatch' 41 | require 'action_controller' 42 | require 'rspec/rails' 43 | 44 | require 'taskinator' 45 | 46 | Taskinator.configure do |config| 47 | 48 | # use active support for instrumentation 49 | config.instrumenter = ActiveSupport::Notifications 50 | 51 | # use a "null stream" for logging 52 | config.logger = Logger.new(File::NULL) 53 | 54 | end 55 | 56 | # require supporting files with custom matchers and macros, etc 57 | Dir[File.expand_path("../support/**/*.rb", __FILE__)].each {|f| require f } 58 | 59 | RSpec.configure do |config| 60 | 61 | config.expect_with :rspec do |c| 62 | c.syntax = [:should, :expect] 63 | end 64 | 65 | config.mock_with :rspec do |c| 66 | c.syntax = :expect 67 | end 68 | 69 | config.order = :random 70 | config.fail_fast = (ENV["FAIL_FAST"] == 1) 71 | 72 | config.before(:each) do 73 | Taskinator.queue_adapter = :test_queue 74 | end 75 | 76 | config.before(:each, :redis => true) do 77 | Taskinator.redis = { :namespace => "taskinator:test:#{SecureRandom.uuid}" } 78 | end 79 | 80 | config.before(:each, :sidekiq => true) do 81 | Sidekiq::Worker.clear_all 82 | end 83 | 84 | config.before(:each, :delayed_job => true) do 85 | Delayed::Job.clear_all 86 | end 87 | 88 | end 89 | 90 | # require examples, must happen after configure 91 | Dir[File.expand_path("../examples/**/*.rb", __FILE__)].each {|f| require f } 92 | 93 | def recursively_enumerate_tasks(tasks, &block) 94 | tasks.each do |task| 95 | if task.is_a?(Taskinator::Task::SubProcess) 96 | recursively_enumerate_tasks(task.sub_process.tasks, &block) 97 | else 98 | yield task 99 | end 100 | end 101 | end 102 | -------------------------------------------------------------------------------- /spec/support/delayed_job.rb: -------------------------------------------------------------------------------- 1 | # mock version of a delayed backend 2 | module Delayed 3 | module Job 4 | def self.queue 5 | @queue ||= [] 6 | end 7 | 8 | def self.clear_all 9 | @queue = [] 10 | end 11 | 12 | def self.enqueue(*args) 13 | queue << args 14 | end 15 | 16 | # NOTE: expects only one job in the queue, so don't forget to clear down the fake queue before each spec 17 | def self.contains?(job_class, args=nil, queue_name=:default) 18 | entry = queue.first 19 | entry && 20 | (entry.first.class == job_class) && 21 | (entry.first.to_a == [*args]) && 22 | (entry.last[:queue] == queue_name) 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /spec/support/mock_definition.rb: -------------------------------------------------------------------------------- 1 | module MockDefinition 2 | 3 | class << self 4 | def create(queue=nil) 5 | 6 | definition = Module.new do 7 | extend Taskinator::Definition 8 | 9 | define_process :foo_hash do 10 | # empty on purpose 11 | end 12 | end 13 | 14 | definition.queue = queue 15 | 16 | # create a constant, so that the mock definition isn't anonymous 17 | Object.const_set("Mock#{SecureRandom.hex}Definition", definition) 18 | 19 | end 20 | end 21 | 22 | end 23 | -------------------------------------------------------------------------------- /spec/support/mock_model.rb: -------------------------------------------------------------------------------- 1 | class MockModel 2 | 3 | attr_reader :model_id 4 | attr_reader :model_type 5 | 6 | def initialize 7 | @model_id = 1 8 | @model_type = 'TypeX' 9 | end 10 | 11 | def global_id 12 | { :model_id => model_id, :model_type => model_type } 13 | end 14 | 15 | def find 16 | end 17 | 18 | end 19 | -------------------------------------------------------------------------------- /spec/support/process_methods.rb: -------------------------------------------------------------------------------- 1 | module ProcessMethods 2 | def enqueue 3 | end 4 | 5 | def cancel 6 | end 7 | 8 | def start 9 | end 10 | 11 | def pause 12 | end 13 | 14 | def resume 15 | end 16 | 17 | def complete 18 | end 19 | 20 | def fail(error) 21 | end 22 | 23 | def task_completed(task) 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /spec/support/sidekiq_matchers.rb: -------------------------------------------------------------------------------- 1 | module RSpec 2 | module Sidekiq 3 | module Matchers 4 | 5 | # 6 | # original version here: 7 | # https://github.com/philostler/rspec-sidekiq/blob/develop/lib/rspec/sidekiq/matchers/be_processed_in.rb 8 | # 9 | # this matcher needs to read off the actual queue name used, from the job array 10 | # instead of reading off the job classes configuration 11 | # 12 | 13 | def be_processed_in_x(expected_queue) 14 | BeProcessedInX.new expected_queue 15 | end 16 | 17 | class BeProcessedInX 18 | def initialize(expected_queue) 19 | @expected_queue = expected_queue 20 | end 21 | 22 | def description 23 | "be processed in the \"#{@expected_queue}\" queue" 24 | end 25 | 26 | def failure_message 27 | "expected #{@klass} to be processed in the \"#{@expected_queue}\" queue but got \"#{@actual}\"" 28 | end 29 | 30 | # NOTE: expects only one job in the queue, so don't forget to clear down the fake queue before each spec 31 | def matches?(job) 32 | @klass = job.is_a?(Class) ? job : job.class 33 | entry = @klass.jobs.first 34 | entry && (entry['queue'] == @expected_queue.to_s) 35 | end 36 | 37 | def failure_message_when_negated 38 | "expected #{@klass} to not be processed in the \"#{@expected_queue}\" queue" 39 | end 40 | end 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /spec/support/spec_support.rb: -------------------------------------------------------------------------------- 1 | module SpecSupport 2 | class Block 3 | def to_proc 4 | lambda { |*args| 5 | call(*args) 6 | } 7 | end 8 | 9 | # the call method must be provided by in specs 10 | # E.g. using `expect(mock_block_instance).to receive(:call)` to assert that the "block" gets called 11 | def call 12 | raise NotImplementedError, "Expecting `call` method to have an expectation defined to assert." 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/support/task_methods.rb: -------------------------------------------------------------------------------- 1 | module TaskMethods 2 | def enqueue 3 | end 4 | 5 | def start 6 | end 7 | 8 | def complete 9 | end 10 | 11 | def fail(error) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/support/test_definition.rb: -------------------------------------------------------------------------------- 1 | module TestDefinition 2 | extend Taskinator::Definition 3 | 4 | def do_task(*args) 5 | end 6 | 7 | end 8 | -------------------------------------------------------------------------------- /spec/support/test_flow.rb: -------------------------------------------------------------------------------- 1 | module TestFlow 2 | extend Taskinator::Definition 3 | 4 | define_process :some_arg1, :some_arg2 do 5 | 6 | # TODO: add support for "continue_on_error" 7 | task :error_task, :continue_on_error => true 8 | 9 | task :the_task 10 | 11 | for_each :iterator do 12 | task :the_task 13 | end 14 | 15 | for_each :iterator, :sub_option => 1 do 16 | task :the_task 17 | end 18 | 19 | sequential do 20 | task :the_task 21 | task :the_task 22 | task :the_task 23 | end 24 | 25 | task :the_task 26 | 27 | concurrent do 28 | 20.times do |i| 29 | task :the_task 30 | end 31 | task :the_task 32 | end 33 | 34 | task :the_task 35 | 36 | # invoke the specified sub process 37 | sub_process TestSubFlow 38 | 39 | job TestWorkerJob 40 | end 41 | 42 | def error_task(*args) 43 | raise "It's a huge problem!" 44 | end 45 | 46 | # note: arg1 and arg2 are passed in all the way from the 47 | # definition#create_process method 48 | def iterator(arg1, arg2, options={}) 49 | 3.times do |i| 50 | yield [arg1, arg2, i] 51 | end 52 | end 53 | 54 | def the_task(*args) 55 | t = rand(1..11) 56 | Taskinator.logger.info "Executing task '#{task}' with [#{args}] for #{t} secs..." 57 | sleep 1 # 1 58 | end 59 | 60 | module TestSubFlow 61 | extend Taskinator::Definition 62 | 63 | define_process :some_arg1, :some_arg2 do 64 | task :the_task 65 | task :the_task 66 | task :the_task 67 | end 68 | 69 | def the_task(*args) 70 | t = rand(1..11) 71 | Taskinator.logger.info "Executing sub task '#{task}' with [#{args}] for #{t} secs..." 72 | sleep 1 # t 73 | end 74 | end 75 | 76 | module TestWorkerJob 77 | def self.perform(*args) 78 | end 79 | 80 | def perform(*args) 81 | end 82 | end 83 | 84 | end 85 | -------------------------------------------------------------------------------- /spec/support/test_flows.rb: -------------------------------------------------------------------------------- 1 | module TestFlows 2 | 3 | module Worker 4 | def self.perform(*args) 5 | # nop 6 | end 7 | end 8 | 9 | module Support 10 | 11 | def iterator(task_count, *args) 12 | task_count.times do |i| 13 | yield i, *args 14 | end 15 | end 16 | 17 | def do_task(*args) 18 | Taskinator.logger.info(">>> Executing task do_task [#{uuid}]...") 19 | end 20 | 21 | # just create lots of these, so it's easy to see which task 22 | # corresponds with each method when debugging specs 23 | 20.times do |i| 24 | define_method "task_#{i}" do |*args| 25 | Taskinator.logger.info(">>> Executing task #{__method__} [#{uuid}]...") 26 | end 27 | end 28 | 29 | end 30 | 31 | module Task 32 | extend Taskinator::Definition 33 | include Support 34 | 35 | define_process :task_count do 36 | for_each :iterator do 37 | task :do_task, :queue => :foo 38 | end 39 | end 40 | 41 | end 42 | 43 | module Job 44 | extend Taskinator::Definition 45 | include Support 46 | 47 | define_process :task_count do 48 | for_each :iterator do 49 | job Worker 50 | end 51 | end 52 | 53 | end 54 | 55 | module SubProcess 56 | extend Taskinator::Definition 57 | include Support 58 | 59 | define_process :task_count do 60 | sub_process Task 61 | end 62 | 63 | end 64 | 65 | module Sequential 66 | extend Taskinator::Definition 67 | include Support 68 | 69 | define_process :task_count do 70 | sequential do 71 | for_each :iterator do 72 | task :do_task 73 | end 74 | end 75 | end 76 | 77 | end 78 | 79 | module Concurrent 80 | extend Taskinator::Definition 81 | include Support 82 | 83 | define_process :task_count do 84 | concurrent do 85 | for_each :iterator do 86 | task :do_task 87 | end 88 | end 89 | end 90 | 91 | end 92 | 93 | module EmptySequentialProcessTest 94 | extend Taskinator::Definition 95 | include Support 96 | 97 | define_process do 98 | 99 | task :task_0 100 | 101 | sequential do 102 | # NB: empty! 103 | end 104 | 105 | sequential do 106 | task :task_1 107 | end 108 | 109 | task :task_2 110 | 111 | end 112 | end 113 | 114 | module EmptyConcurrentProcessTest 115 | extend Taskinator::Definition 116 | include Support 117 | 118 | define_process do 119 | 120 | task :task_0 121 | 122 | concurrent do 123 | # NB: empty! 124 | end 125 | 126 | concurrent do 127 | task :task_1 128 | end 129 | 130 | task :task_2 131 | 132 | end 133 | end 134 | 135 | module NestedTask 136 | extend Taskinator::Definition 137 | include Support 138 | 139 | define_process :task_count do 140 | task :task_1 141 | 142 | concurrent do 143 | task :task_2 144 | task :task_3 145 | 146 | sequential do 147 | task :task_4 148 | task :task_5 149 | 150 | concurrent do 151 | task :task_6 152 | task :task_7 153 | 154 | sequential do 155 | task :task_8 156 | task :task_9 157 | 158 | end 159 | 160 | task :task_10 161 | end 162 | 163 | task :task_11 164 | end 165 | 166 | task :task_12 167 | end 168 | 169 | task :task_13 170 | end 171 | end 172 | 173 | end 174 | -------------------------------------------------------------------------------- /spec/support/test_instrumenter.rb: -------------------------------------------------------------------------------- 1 | class TestInstrumenter 2 | 3 | class << self 4 | 5 | def subscribe(callback, filter=nil, &block) 6 | 7 | # create test instrumenter instance 8 | instrumenter = TestInstrumenter.new do |name, payload| 9 | if filter 10 | callback.call(name, payload) if name =~ filter 11 | else 12 | callback.call(name, payload) 13 | end 14 | end 15 | 16 | # hook up this instrumenter in the context of the spec 17 | # (assuming called from RSpec binding) 18 | spec_binding = block.binding.eval('self') 19 | spec_binding.instance_exec do 20 | allow(Taskinator).to receive(:instrumenter).and_return(instrumenter) 21 | end 22 | 23 | yield 24 | end 25 | 26 | end 27 | 28 | attr_reader :callback 29 | 30 | def initialize(&block) 31 | @callback = block 32 | end 33 | 34 | def instrument(event, payload={}) 35 | @callback.call(event, payload) 36 | yield(payload) if block_given? 37 | end 38 | 39 | end 40 | -------------------------------------------------------------------------------- /spec/support/test_job.rb: -------------------------------------------------------------------------------- 1 | module TestJob 2 | def self.perform(*args) 3 | end 4 | end 5 | 6 | class TestJobClass 7 | def perform(*args) 8 | end 9 | end 10 | 11 | module TestJobModule 12 | def self.perform(*args) 13 | end 14 | end 15 | 16 | class TestJobClassNoArgs 17 | def perform 18 | end 19 | end 20 | 21 | module TestJobModuleNoArgs 22 | def self.perform 23 | end 24 | end 25 | 26 | module TestJobError 27 | def self.perform 28 | raise ArgumentError 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /spec/support/test_job_task.rb: -------------------------------------------------------------------------------- 1 | class TestJobTask < Taskinator::Task::Job 2 | end 3 | -------------------------------------------------------------------------------- /spec/support/test_process.rb: -------------------------------------------------------------------------------- 1 | class TestProcess < Taskinator::Process 2 | end 3 | -------------------------------------------------------------------------------- /spec/support/test_queue.rb: -------------------------------------------------------------------------------- 1 | module Taskinator 2 | module Queues 3 | 4 | def self.create_test_queue_adapter(config={}) 5 | TestQueueAdapter.new(config) 6 | end 7 | 8 | def self.create_test_queue_worker_adapter(config={}) 9 | TestQueueWorkerAdapter.new(config) 10 | end 11 | 12 | # 13 | # this is a no-op adapter, it tracks enqueued processes and tasks 14 | # 15 | class TestQueueAdapter 16 | 17 | def initialize(config={}) 18 | clear 19 | end 20 | 21 | def enqueue_create_process(definition, uuid, args) 22 | @processes << [definition, uuid, args] 23 | end 24 | 25 | def enqueue_task(task) 26 | @tasks << task 27 | end 28 | 29 | # helpers 30 | 31 | attr_reader :processes 32 | attr_reader :tasks 33 | 34 | def clear 35 | @processes = [] 36 | @tasks = [] 37 | end 38 | 39 | def empty? 40 | @processes.empty? && @tasks.empty? 41 | end 42 | 43 | end 44 | 45 | # 46 | # this is a "synchronous" implementation for use in testing 47 | # 48 | class TestQueueWorkerAdapter < TestQueueAdapter 49 | 50 | def enqueue_create_process(definition, uuid, args) 51 | super 52 | invoke do 53 | Taskinator::CreateProcessWorker.new(definition.name, uuid, args).perform 54 | end 55 | end 56 | 57 | def enqueue_task(task) 58 | super 59 | invoke do 60 | Taskinator::TaskWorker.new(task.uuid).perform 61 | end 62 | end 63 | 64 | private 65 | 66 | def invoke(&block) 67 | block.call 68 | end 69 | 70 | end 71 | 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /spec/support/test_step_task.rb: -------------------------------------------------------------------------------- 1 | class TestStepTask < Taskinator::Task::Step 2 | end 3 | -------------------------------------------------------------------------------- /spec/support/test_subprocess_task.rb: -------------------------------------------------------------------------------- 1 | class TestSubProcessTask < Taskinator::Task::SubProcess 2 | end 3 | -------------------------------------------------------------------------------- /spec/support/test_task.rb: -------------------------------------------------------------------------------- 1 | class TestTask < Taskinator::Task 2 | end 3 | -------------------------------------------------------------------------------- /spec/taskinator/api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Api, :redis => true do 4 | 5 | describe Taskinator::Api::Processes do 6 | 7 | it { expect(subject).to be_a(::Enumerable) } 8 | 9 | describe "#each" do 10 | it "does not enumerate when there aren't any processes" do 11 | block = SpecSupport::Block.new 12 | expect(block).to_not receive(:call) 13 | subject.each(&block) 14 | end 15 | 16 | it "it enumerates processes" do 17 | allow_any_instance_of(Process).to receive(:fetch) {} 18 | 19 | Taskinator.redis do |conn| 20 | conn.multi do |transaction| 21 | 3.times {|i| transaction.sadd(Taskinator::Persistence.processes_list_key, i) } 22 | end 23 | end 24 | 25 | block = SpecSupport::Block.new 26 | expect(block).to receive(:call).exactly(3).times 27 | 28 | subject.each(&block) 29 | end 30 | end 31 | 32 | describe "#size" do 33 | it { expect(subject.size).to eq(0) } 34 | 35 | it "yields the number of processes" do 36 | Taskinator.redis do |conn| 37 | conn.multi do |transaction| 38 | 3.times {|i| transaction.sadd(Taskinator::Persistence.processes_list_key, i) } 39 | end 40 | end 41 | 42 | expect(subject.size).to eq(3) 43 | end 44 | end 45 | end 46 | 47 | describe "#find_process" do 48 | it { 49 | # fetch method is covered by persistence spec 50 | expect(Taskinator::Process).to receive(:fetch) {} 51 | subject.find_process 'foo:bar:process' 52 | } 53 | end 54 | 55 | describe "#find_task" do 56 | it { 57 | # fetch method is covered by persistence spec 58 | expect(Taskinator::Task).to receive(:fetch) {} 59 | subject.find_task 'foo:bar:process:baz:task' 60 | } 61 | end 62 | 63 | end 64 | -------------------------------------------------------------------------------- /spec/taskinator/builder_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Builder do 4 | 5 | let(:definition) do 6 | Module.new do 7 | extend Taskinator::Definition 8 | 9 | def iterator_method(*); end 10 | def task_method(*); end 11 | end 12 | end 13 | 14 | let(:process) { 15 | Class.new(Taskinator::Process).new(definition) 16 | } 17 | 18 | let(:args) { [:arg1, :arg2] } 19 | let(:builder_options) { {:option1 => 1, :another => false} } 20 | let(:options) { { :bar => :baz } } 21 | 22 | let(:block) { SpecSupport::Block.new } 23 | 24 | let(:define_block) { 25 | the_block = block 26 | Proc.new {|*args| the_block.call } 27 | } 28 | 29 | subject { Taskinator::Builder.new(process, definition, *[*args, builder_options]) } 30 | 31 | it "assign attributes" do 32 | expect(subject.process).to eq(process) 33 | expect(subject.definition).to eq(definition) 34 | expect(subject.args).to eq(args) 35 | expect(subject.builder_options).to eq(builder_options) 36 | end 37 | 38 | describe "#option?" do 39 | it "invokes supplied block for 'option1' option" do 40 | expect(block).to receive(:call) 41 | subject.option?(:option1, &define_block) 42 | end 43 | 44 | it "does not invoke supplied block for 'another' option" do 45 | expect(block).to_not receive(:call) 46 | subject.option?(:another, &define_block) 47 | end 48 | 49 | it "does not invoke supplied block for an unspecified option" do 50 | expect(block).to_not receive(:call) 51 | subject.option?(:unspecified, &define_block) 52 | end 53 | end 54 | 55 | describe "#sequential" do 56 | it "invokes supplied block" do 57 | expect(block).to receive(:call) 58 | subject.sequential(&define_block) 59 | end 60 | 61 | it "creates a sequential process" do 62 | allow(block).to receive(:call) 63 | expect(Taskinator::Process).to receive(:define_sequential_process_for).with(definition, {}).and_call_original 64 | subject.sequential(&define_block) 65 | end 66 | 67 | it "fails if block isn't given" do 68 | expect { 69 | subject.sequential 70 | }.to raise_error(ArgumentError) 71 | end 72 | 73 | it "includes options" do 74 | allow(block).to receive(:call) 75 | expect(Taskinator::Process).to receive(:define_sequential_process_for).with(definition, options).and_call_original 76 | subject.sequential(options, &define_block) 77 | end 78 | 79 | it "adds sub-process task" do 80 | block = Proc.new {|p| 81 | p.task :task_method 82 | } 83 | expect(process.tasks).to be_empty 84 | subject.sequential(options, &block) 85 | expect(process.tasks).to_not be_empty 86 | end 87 | 88 | it "ignores sub-processes without tasks" do 89 | allow(block).to receive(:call) 90 | expect(process.tasks).to be_empty 91 | subject.sequential(options, &define_block) 92 | expect(process.tasks).to be_empty 93 | end 94 | end 95 | 96 | describe "#concurrent" do 97 | it "invokes supplied block" do 98 | expect(block).to receive(:call) 99 | subject.concurrent(&define_block) 100 | end 101 | 102 | it "creates a concurrent process" do 103 | allow(block).to receive(:call) 104 | expect(Taskinator::Process).to receive(:define_concurrent_process_for).with(definition, Taskinator::CompleteOn::First, {}).and_call_original 105 | subject.concurrent(Taskinator::CompleteOn::First, &define_block) 106 | end 107 | 108 | it "fails if block isn't given" do 109 | expect { 110 | subject.concurrent 111 | }.to raise_error(ArgumentError) 112 | end 113 | 114 | it "includes options" do 115 | allow(block).to receive(:call) 116 | expect(Taskinator::Process).to receive(:define_concurrent_process_for).with(definition, Taskinator::CompleteOn::First, options).and_call_original 117 | subject.concurrent(Taskinator::CompleteOn::First, options, &define_block) 118 | end 119 | 120 | it "adds sub-process task" do 121 | block = Proc.new {|p| 122 | p.task :task_method 123 | } 124 | expect(process.tasks).to be_empty 125 | subject.sequential(options, &block) 126 | expect(process.tasks).to_not be_empty 127 | end 128 | 129 | it "ignores sub-processes without tasks" do 130 | allow(block).to receive(:call) 131 | expect(process.tasks).to be_empty 132 | subject.sequential(options, &define_block) 133 | expect(process.tasks).to be_empty 134 | end 135 | end 136 | 137 | describe "#for_each" do 138 | it "creates tasks for each returned item" do 139 | # the definition is mixed into the eigen class of Executor 140 | 141 | # HACK: replace the internal executor instance 142 | executor = Taskinator::Executor.new(definition) 143 | subject.instance_eval do 144 | @executor = executor 145 | end 146 | 147 | expect(executor).to receive(:iterator_method).with(*args) do |*a, &block| 148 | 3.times(&block) 149 | end 150 | 151 | expect(block).to receive(:call).exactly(3).times 152 | 153 | subject.for_each(:iterator_method, &define_block) 154 | end 155 | 156 | it "fails if iterator method is nil" do 157 | expect { 158 | subject.for_each(nil, &define_block) 159 | }.to raise_error(ArgumentError) 160 | end 161 | 162 | it "fails if iterator method is not defined" do 163 | expect { 164 | subject.for_each(:undefined_iterator, &define_block) 165 | }.to raise_error(NoMethodError) 166 | end 167 | 168 | it "fails if block isn't given" do 169 | expect { 170 | subject.for_each(nil) 171 | }.to raise_error(ArgumentError) 172 | end 173 | 174 | it "calls the iterator method, adding specified options" do 175 | executor = Taskinator::Executor.new(definition) 176 | 177 | subject.instance_eval do 178 | @executor = executor 179 | end 180 | 181 | expect(executor).to receive(:iterator_method).with(*[*args, :sub_option => 1]) do |*a, &block| 182 | 3.times(&block) 183 | end 184 | 185 | expect(block).to receive(:call).exactly(3).times 186 | 187 | subject.for_each(:iterator_method, :sub_option => 1, &define_block) 188 | end 189 | end 190 | 191 | # NOTE: #transform is an alias for #for_each 192 | 193 | describe "#task" do 194 | it "creates a task" do 195 | expect(Taskinator::Task).to receive(:define_step_task).with(process, :task_method, args, builder_options) 196 | subject.task(:task_method) 197 | end 198 | 199 | it "fails if task method is nil" do 200 | expect { 201 | subject.task(nil) 202 | }.to raise_error(ArgumentError) 203 | end 204 | 205 | it "fails if task method is not defined" do 206 | expect { 207 | subject.task(:undefined) 208 | }.to raise_error(NoMethodError) 209 | end 210 | 211 | it "includes options" do 212 | expect(Taskinator::Task).to receive(:define_step_task).with(process, :task_method, args, builder_options.merge(options)) 213 | subject.task(:task_method, options) 214 | end 215 | end 216 | 217 | describe "#job" do 218 | it "creates a job" do 219 | job = double('job', :perform => true) 220 | expect(Taskinator::Task).to receive(:define_job_task).with(process, job, args, builder_options) 221 | subject.job(job) 222 | end 223 | 224 | it "fails if job module is nil" do 225 | expect { 226 | subject.job(nil) 227 | }.to raise_error(ArgumentError) 228 | end 229 | 230 | # ok, fuzzy logic to determine what is ia job here! 231 | it "fails if job module is not a job" do 232 | expect { 233 | subject.job(double('job', :methods => [], :instance_methods => [])) 234 | }.to raise_error(ArgumentError) 235 | end 236 | 237 | it "includes options" do 238 | job = double('job', :perform => true) 239 | expect(Taskinator::Task).to receive(:define_job_task).with(process, job, args, builder_options.merge(options)) 240 | subject.job(job, options) 241 | end 242 | end 243 | 244 | describe "#sub_process" do 245 | let(:sub_definition) do 246 | Module.new do 247 | extend Taskinator::Definition 248 | 249 | define_process :some_arg1, :some_arg2, :some_arg3 do 250 | end 251 | end 252 | end 253 | 254 | it "creates a sub process" do 255 | expect(sub_definition).to receive(:create_sub_process).with(*args, builder_options).and_call_original 256 | subject.sub_process(sub_definition) 257 | end 258 | 259 | it "creates a sub process task" do 260 | sub_process = sub_definition.create_process(:argX, :argY, :argZ) 261 | allow(sub_definition).to receive(:create_sub_process) { sub_process } 262 | expect(Taskinator::Task).to receive(:define_sub_process_task).with(process, sub_process, builder_options) 263 | subject.sub_process(sub_definition) 264 | end 265 | 266 | it "includes options" do 267 | expect(sub_definition).to receive(:create_sub_process).with(*args, builder_options.merge(options)).and_call_original 268 | subject.sub_process(sub_definition, options) 269 | end 270 | 271 | it "adds sub-process task" do 272 | block = Proc.new {|p| 273 | p.task :task_method 274 | } 275 | expect(process.tasks).to be_empty 276 | subject.sequential(options, &block) 277 | expect(process.tasks).to_not be_empty 278 | end 279 | 280 | it "ignores sub-processes without tasks" do 281 | allow(block).to receive(:call) 282 | expect(process.tasks).to be_empty 283 | subject.sequential(options, &define_block) 284 | expect(process.tasks).to be_empty 285 | end 286 | end 287 | 288 | end 289 | -------------------------------------------------------------------------------- /spec/taskinator/complex_process_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe TestFlow, :redis => true do 4 | it "should persist and retrieve" do 5 | processA = TestFlow.create_process(:arg1, :arg2) 6 | 7 | processB = Taskinator::Process.fetch(processA.uuid) 8 | 9 | expect(processB.uuid).to eq(processA.uuid) 10 | expect(processB.definition).to eq(processA.definition) 11 | expect(processB.options).to eq(processA.options) 12 | 13 | expect(processB.tasks.count).to eq(processA.tasks.count) 14 | 15 | tasks = processA.tasks.zip(processB.tasks) 16 | 17 | tasks.each do |(taskB, taskA)| 18 | expect(taskA.process).to eq(taskB.process) 19 | expect(taskA.uuid).to eq(taskB.uuid) 20 | expect(taskA.options).to eq(taskB.options) 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/taskinator/create_process_worker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::CreateProcessWorker do 4 | 5 | let(:definition) { MockDefinition.create } 6 | let(:uuid) { Taskinator.generate_uuid } 7 | let(:args) { [{:foo => :bar}] } 8 | 9 | subject { Taskinator::CreateProcessWorker.new(definition.name, uuid, Taskinator::Persistence.serialize(args)) } 10 | 11 | describe "#initialize" do 12 | it { 13 | expect(subject.definition).to eq(definition) 14 | } 15 | 16 | it { 17 | Taskinator::CreateProcessWorker.new(definition.name, uuid, Taskinator::Persistence.serialize(args)) 18 | expect(subject.definition).to eq(definition) 19 | } 20 | 21 | it { 22 | MockDefinition.const_set(definition.name, definition) 23 | Taskinator::CreateProcessWorker.new("MockDefinition::#{definition.name}", uuid, Taskinator::Persistence.serialize(args)) 24 | expect(subject.definition).to eq(definition) 25 | } 26 | 27 | it { 28 | expect { 29 | Taskinator::CreateProcessWorker.new("NonExistent", uuid, Taskinator::Persistence.serialize(args)) 30 | }.to raise_error(NameError) 31 | } 32 | 33 | it { 34 | expect(subject.uuid).to eq(uuid) 35 | } 36 | 37 | it { 38 | expect(subject.args).to eq(args) 39 | } 40 | end 41 | 42 | describe "#perform" do 43 | describe "create the process" do 44 | it "with no arguments" do 45 | process_args = [{:uuid => uuid}] 46 | args = Taskinator::Persistence.serialize([]) 47 | 48 | expect(definition).to receive(:_create_process_).with(false, *process_args).and_return(double('process', :enqueue! => nil)) 49 | 50 | Taskinator::CreateProcessWorker.new(definition.name, uuid, args).perform 51 | end 52 | 53 | it "with arguments" do 54 | process_args = [:foo, :bar, {:uuid => uuid}] 55 | serialized_args = Taskinator::Persistence.serialize([:foo, :bar]) 56 | 57 | expect(definition).to receive(:_create_process_).with(false, *process_args).and_return(double('process', :enqueue! => nil)) 58 | 59 | Taskinator::CreateProcessWorker.new(definition.name, uuid, serialized_args).perform 60 | end 61 | 62 | it "with options" do 63 | process_args = [{:foo => :bar, :uuid => uuid}] 64 | serialized_args = Taskinator::Persistence.serialize([{:foo => :bar}]) 65 | 66 | expect(definition).to receive(:_create_process_).with(false, *process_args).and_return(double('process', :enqueue! => nil)) 67 | 68 | Taskinator::CreateProcessWorker.new(definition.name, uuid, serialized_args).perform 69 | end 70 | 71 | it "with arguments and options" do 72 | process_args = [:foo, {:bar => :baz, :uuid => uuid}] 73 | serialized_args = Taskinator::Persistence.serialize([:foo, {:bar => :baz}]) 74 | 75 | expect(definition).to receive(:_create_process_).with(false, *process_args).and_return(double('process', :enqueue! => nil)) 76 | 77 | Taskinator::CreateProcessWorker.new(definition.name, uuid, serialized_args).perform 78 | end 79 | end 80 | 81 | it "should enqueue the process" do 82 | expect_any_instance_of(Taskinator::Process).to receive(:enqueue!) 83 | subject.perform 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /spec/taskinator/definition_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Definition do 4 | 5 | subject do 6 | Module.new do 7 | extend Taskinator::Definition 8 | end 9 | end 10 | 11 | it "should respond to #define_process" do 12 | expect(subject).to respond_to(:define_process) 13 | end 14 | 15 | it "should have a #create_process method" do 16 | expect(subject).to respond_to(:create_process) 17 | end 18 | 19 | describe "#define_process" do 20 | it "should define a #create_process method" do 21 | expect(subject).to respond_to(:create_process) 22 | end 23 | 24 | it "should not invoke the given block" do 25 | block = SpecSupport::Block.new 26 | expect(block).to_not receive(:call) 27 | subject.define_process(&block) 28 | end 29 | 30 | it "should raise ProcessAlreadyDefinedError error if already defined" do 31 | subject.define_process 32 | expect { 33 | subject.define_process 34 | }.to raise_error(Taskinator::Definition::ProcessAlreadyDefinedError) 35 | end 36 | 37 | it "should create a sequential process" do 38 | subject.define_process {} 39 | expect(subject.create_process).to be_a(Taskinator::Process::Sequential) 40 | end 41 | end 42 | 43 | describe "#define_sequential_process" do 44 | it "should define a #define_sequential_process method" do 45 | expect(subject).to respond_to(:define_sequential_process) 46 | end 47 | 48 | it "should not invoke the given block" do 49 | block = SpecSupport::Block.new 50 | expect(block).to_not receive(:call) 51 | subject.define_sequential_process(&block) 52 | end 53 | 54 | it "should raise ProcessAlreadyDefinedError error if already defined" do 55 | subject.define_sequential_process 56 | expect { 57 | subject.define_sequential_process 58 | }.to raise_error(Taskinator::Definition::ProcessAlreadyDefinedError) 59 | end 60 | 61 | it "should create a sequential process" do 62 | subject.define_sequential_process {} 63 | expect(subject.create_process).to be_a(Taskinator::Process::Sequential) 64 | end 65 | end 66 | 67 | describe "#define_concurrent_process" do 68 | it "should define a #define_concurrent_process method" do 69 | subject.define_concurrent_process 70 | expect(subject).to respond_to(:define_concurrent_process) 71 | end 72 | 73 | it "should not invoke the given block" do 74 | block = SpecSupport::Block.new 75 | expect(block).to_not receive(:call) 76 | subject.define_concurrent_process(&block) 77 | end 78 | 79 | it "should raise ProcessAlreadyDefinedError error if already defined" do 80 | subject.define_concurrent_process 81 | expect { 82 | subject.define_concurrent_process 83 | }.to raise_error(Taskinator::Definition::ProcessAlreadyDefinedError) 84 | end 85 | 86 | it "should create a concurrent process" do 87 | subject.define_concurrent_process {} 88 | expect(subject.create_process).to be_a(Taskinator::Process::Concurrent) 89 | end 90 | end 91 | 92 | describe "#create_process" do 93 | it "raises ProcessUndefinedError" do 94 | expect { 95 | subject.create_process 96 | }.to raise_error(Taskinator::Definition::ProcessUndefinedError) 97 | end 98 | 99 | it "returns a process" do 100 | block = SpecSupport::Block.new 101 | allow(block).to receive(:to_proc) { 102 | Proc.new {|*args| } 103 | } 104 | subject.define_process(&block) 105 | 106 | expect(subject.create_process).to be_a(Taskinator::Process) 107 | end 108 | 109 | it "handles error" do 110 | subject.define_process do 111 | raise ArgumentError 112 | end 113 | 114 | expect { 115 | subject.create_process 116 | }.to raise_error(ArgumentError) 117 | end 118 | 119 | it "checks defined arguments provided" do 120 | subject.define_process :arg1, :arg2 do 121 | end 122 | 123 | expect { 124 | subject.create_process 125 | }.to raise_error(ArgumentError) 126 | 127 | expect { 128 | subject.create_process :foo 129 | }.to raise_error(ArgumentError) 130 | 131 | expect { 132 | subject.create_process :foo, :bar 133 | }.not_to raise_error 134 | 135 | expect { 136 | subject.create_process :foo, :bar, :baz 137 | }.not_to raise_error 138 | end 139 | 140 | it "defaults the scope to :shared" do 141 | block = SpecSupport::Block.new 142 | allow(block).to receive(:to_proc) { 143 | Proc.new {|*args| } 144 | } 145 | subject.define_process(&block) 146 | 147 | expect(subject.create_process.scope).to eq(:shared) 148 | end 149 | 150 | it "sets the scope" do 151 | block = SpecSupport::Block.new 152 | allow(block).to receive(:to_proc) { 153 | Proc.new {|*args| } 154 | } 155 | subject.define_process(&block) 156 | 157 | expect(subject.create_process(:scope => :foo).scope).to eq(:foo) 158 | end 159 | 160 | it "receives options" do 161 | block = SpecSupport::Block.new 162 | allow(block).to receive(:to_proc) { 163 | Proc.new {|*args| } 164 | } 165 | subject.define_process(&block) 166 | 167 | process = subject.create_process(:foo => :bar) 168 | expect(process.options).to eq(:foo => :bar) 169 | end 170 | 171 | it "invokes the given block in the context of a ProcessBuilder" do 172 | block = SpecSupport::Block.new 173 | expect(block).to receive(:call) 174 | 175 | subject.define_process do 176 | 177 | # make sure we get here! 178 | block.call 179 | 180 | # we should be in the context of the Builder 181 | # so methods such as concurrent, for_each and task 182 | # should be directly available 183 | raise RuntimeError unless self.respond_to?(:task) 184 | 185 | end 186 | 187 | # if an error is raised, then the context was incorrect 188 | expect { 189 | subject.create_process 190 | }.not_to raise_error 191 | end 192 | 193 | context "is instrumented" do 194 | subject { MockDefinition.create } 195 | 196 | it "for create process" do 197 | instrumentation_block = SpecSupport::Block.new 198 | expect(instrumentation_block).to receive(:call) do |*args| 199 | expect(args.first).to eq('taskinator.process.created') 200 | end 201 | 202 | TestInstrumenter.subscribe(instrumentation_block, /taskinator.process.created/) do 203 | subject.create_process :foo 204 | end 205 | end 206 | 207 | it "for save process" do 208 | instrumentation_block = SpecSupport::Block.new 209 | expect(instrumentation_block).to receive(:call) do |*args| 210 | expect(args.first).to eq('taskinator.process.saved') 211 | end 212 | 213 | TestInstrumenter.subscribe(instrumentation_block, /taskinator.process.saved/) do 214 | subject.create_process :foo 215 | end 216 | end 217 | end 218 | end 219 | 220 | describe "#create_process_remotely" do 221 | it "raises ProcessUndefinedError" do 222 | expect { 223 | subject.create_process_remotely 224 | }.to raise_error(Taskinator::Definition::ProcessUndefinedError) 225 | end 226 | 227 | it "returns the process uuid" do 228 | block = SpecSupport::Block.new 229 | allow(block).to receive(:to_proc) { 230 | Proc.new {|*args| } 231 | } 232 | subject.define_process(&block) 233 | 234 | process = subject.create_process_remotely 235 | 236 | expect(process).to_not be_nil 237 | end 238 | 239 | it "enqueues" do 240 | block = SpecSupport::Block.new 241 | allow(block).to receive(:to_proc) { 242 | Proc.new {|*args| } 243 | } 244 | subject.define_process(&block) 245 | 246 | expect(Taskinator.queue).to receive(:enqueue_create_process) 247 | 248 | subject.create_process_remotely 249 | end 250 | end 251 | 252 | describe "#queue" do 253 | it { 254 | expect(subject.queue).to be_nil 255 | } 256 | 257 | it { 258 | subject.queue = :foo 259 | expect(subject.queue).to eq(:foo) 260 | } 261 | end 262 | 263 | end 264 | -------------------------------------------------------------------------------- /spec/taskinator/executor_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Executor do 4 | 5 | let(:definition) do 6 | Module.new do 7 | def method; end 8 | end 9 | end 10 | 11 | let(:task) { double('task') } 12 | subject { Taskinator::Executor.new(definition, task) } 13 | 14 | describe "helpers" do 15 | it "#process_uuid" do 16 | expect(task).to receive(:process_uuid) 17 | subject.process_uuid 18 | end 19 | 20 | it "#uuid" do 21 | expect(task).to receive(:uuid) 22 | subject.uuid 23 | end 24 | 25 | it "#options" do 26 | expect(task).to receive(:options) 27 | subject.options 28 | end 29 | end 30 | 31 | it "should mixin definition" do 32 | expect(subject).to be_a(definition) 33 | end 34 | 35 | it "should mixin definition for the instance only" do 36 | expect(Taskinator::Executor).to_not be_a(definition) 37 | end 38 | 39 | it "should assign definition" do 40 | expect(subject.definition).to eq(definition) 41 | end 42 | 43 | it "should contain definition methods" do 44 | expect(subject).to respond_to(:method) 45 | end 46 | 47 | end 48 | -------------------------------------------------------------------------------- /spec/taskinator/instrumentation_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Instrumentation, :redis => true do 4 | 5 | subject do 6 | klass = Class.new do 7 | include Taskinator::Persistence 8 | include Taskinator::Instrumentation 9 | 10 | def self.base_key 11 | 'base_key' 12 | end 13 | 14 | attr_reader :uuid 15 | attr_reader :options 16 | attr_reader :definition 17 | 18 | def initialize 19 | @uuid = Taskinator.generate_uuid 20 | @options = { :bar => :baz } 21 | @definition = TestDefinition 22 | end 23 | end 24 | 25 | klass.new 26 | end 27 | 28 | describe "#instrument" do 29 | it { 30 | event = 'foo_bar' 31 | 32 | expect(Taskinator.instrumenter).to receive(:instrument).with(event, {}).and_call_original 33 | 34 | block = SpecSupport::Block.new 35 | expect(block).to receive(:call) 36 | 37 | subject.instrument(event, &block) 38 | } 39 | 40 | it { 41 | event = 'foo_bar' 42 | 43 | expect(Taskinator.instrumenter).to receive(:instrument).with(event, {:baz => :qux}).and_call_original 44 | 45 | block = SpecSupport::Block.new 46 | expect(block).to receive(:call) 47 | 48 | subject.instrument(event, :baz => :qux, &block) 49 | } 50 | end 51 | 52 | describe "#enqueued_payload" do 53 | pending 54 | end 55 | 56 | describe "#processing_payload" do 57 | pending 58 | end 59 | 60 | describe "#completed_payload" do 61 | it { 62 | Taskinator.redis do |conn| 63 | conn.hset(subject.key, :process_uuid, subject.uuid) 64 | conn.hmset( 65 | subject.process_key, 66 | [:options, YAML.dump({:foo => :bar})], 67 | [:tasks_count, 100], 68 | [:tasks_processing, 1], 69 | [:tasks_completed, 2], 70 | [:tasks_cancelled, 3], 71 | [:tasks_failed, 4] 72 | ) 73 | end 74 | 75 | expect(subject.completed_payload(:baz => :qux)).to eq( 76 | OpenStruct.new({ 77 | :type => subject.class.name, 78 | :definition => subject.definition.name, 79 | :process_uuid => subject.uuid, 80 | :process_options => {:foo => :bar}, 81 | :uuid => subject.uuid, 82 | :state => :completed, 83 | :options => subject.options, 84 | :percentage_processing => 1.0, 85 | :percentage_completed => 2.0, 86 | :percentage_cancelled => 3.0, 87 | :percentage_failed => 4.0, 88 | :baz => :qux 89 | }) 90 | ) 91 | } 92 | end 93 | 94 | describe "#cancelled_payload" do 95 | pending 96 | end 97 | 98 | describe "#failed_payload" do 99 | pending 100 | end 101 | 102 | end 103 | -------------------------------------------------------------------------------- /spec/taskinator/persistence_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Persistence, :redis => true do 4 | 5 | let(:definition) { TestDefinition } 6 | 7 | describe "class methods" do 8 | subject { 9 | Class.new do 10 | include Taskinator::Persistence 11 | end 12 | } 13 | 14 | describe ".key_for" do 15 | before do 16 | allow(subject).to receive(:base_key) { 'base_key' } 17 | end 18 | 19 | it { 20 | expect(subject.key_for('uuid')).to match(/base_key/) 21 | expect(subject.key_for('uuid')).to match(/uuid/) 22 | } 23 | end 24 | 25 | describe ".fetch" do 26 | before do 27 | allow(subject).to receive(:base_key) { 'base_key' } 28 | end 29 | 30 | it "fetches instance" do 31 | item = double('item') 32 | expect_any_instance_of(Taskinator::Persistence::RedisDeserializationVisitor).to receive(:visit) { item } 33 | expect(subject.fetch('uuid')).to eq(item) 34 | end 35 | 36 | it "fetches instance, and adds to cache" do 37 | cache = {} 38 | allow_any_instance_of(Taskinator::Persistence::RedisDeserializationVisitor).to receive(:visit) { true } 39 | subject.fetch('uuid', cache) 40 | expect(cache.key?(subject.key_for('uuid'))).to be 41 | end 42 | 43 | it "fetches instance from cache" do 44 | item = double('item') 45 | cache = { subject.key_for('uuid') => item } 46 | expect(subject.fetch('uuid', cache)).to eq(item) 47 | end 48 | 49 | it "yields UnknownType" do 50 | Taskinator.redis do |conn| 51 | conn.hmset(*[subject.key_for("foo"), [:type, 'UnknownFoo']]) 52 | end 53 | instance = subject.fetch("foo") 54 | expect(instance).to be_a(Taskinator::Persistence::UnknownType) 55 | expect(instance.type).to eq("UnknownFoo") 56 | end 57 | 58 | describe "for processes" do 59 | let(:process) { TestProcess.new(definition) } 60 | 61 | it { 62 | process.save 63 | expect(TestProcess.fetch(process.uuid)).to eq(process) 64 | } 65 | 66 | describe "unknown definition" do 67 | it "yields UnknownType" do 68 | Taskinator.redis do |conn| 69 | conn.hmset(*[process.key, [:type, TestProcess.name], [:uuid, process.uuid], [:definition, 'UnknownFoo']]) 70 | end 71 | 72 | instance = TestProcess.fetch(process.uuid) 73 | expect(instance.uuid).to eq(process.uuid) 74 | expect(instance.definition).to be_a(Taskinator::Persistence::UnknownType) 75 | expect(instance.definition.type).to eq("UnknownFoo") 76 | end 77 | end 78 | end 79 | 80 | describe "for tasks" do 81 | let(:process) { TestProcess.new(definition) } 82 | let(:task) { TestTask.new(process) } 83 | 84 | it { 85 | process.tasks << task 86 | process.save 87 | 88 | instance = TestTask.fetch(task.uuid) 89 | expect(instance).to eq(task) 90 | expect(instance.process).to eq(process) 91 | } 92 | 93 | describe "unknown job" do 94 | let(:task) { TestJobTask.new(process, TestJob, []) } 95 | 96 | it "yields UnknownType" do 97 | Taskinator.redis do |conn| 98 | conn.hmset(*[task.key, [:type, task.class.name], [:uuid, task.uuid], [:job, 'UnknownBar']]) 99 | end 100 | 101 | instance = TestJobTask.fetch(task.uuid) 102 | expect(instance.uuid).to eq(task.uuid) 103 | expect(instance.job).to be_a(Taskinator::Persistence::UnknownType) 104 | expect(instance.job.type).to eq("UnknownBar") 105 | end 106 | end 107 | 108 | describe "unknown subprocess" do 109 | let(:sub_process) { TestProcess.new(definition) } 110 | let(:task) { TestSubProcessTask.new(process, sub_process) } 111 | 112 | it "yields UnknownType" do 113 | Taskinator.redis do |conn| 114 | conn.multi do |transaction| 115 | transaction.hmset(*[task.key, [:type, task.class.name], [:uuid, task.uuid], [:sub_process, sub_process.uuid]]) 116 | transaction.hmset(*[sub_process.key, [:type, sub_process.class.name], [:uuid, sub_process.uuid], [:definition, 'UnknownBaz']]) 117 | end 118 | end 119 | 120 | instance = TestSubProcessTask.fetch(task.uuid) 121 | expect(instance.uuid).to eq(task.uuid) 122 | expect(instance.sub_process.definition).to be_a(Taskinator::Persistence::UnknownType) 123 | expect(instance.sub_process.definition.type).to eq("UnknownBaz") 124 | end 125 | end 126 | end 127 | end 128 | end 129 | 130 | describe "serialization helpers" do 131 | subject { Taskinator::Persistence } 132 | 133 | describe "#serialize" do 134 | describe "Array" do 135 | it { 136 | expect(subject.serialize([])).to eq(YAML.dump([])) 137 | } 138 | 139 | it { 140 | expect(subject.serialize([1])).to eq(YAML.dump([1])) 141 | } 142 | 143 | it { 144 | expect(subject.serialize(["string"])).to eq(YAML.dump(["string"])) 145 | } 146 | 147 | it { 148 | expect(subject.serialize([MockModel.new])).to eq("---\n- !ruby/object:MockModel\n model_id: 1\n model_type: TypeX\n") 149 | } 150 | end 151 | 152 | describe "Hash" do 153 | it { 154 | expect(subject.serialize({:foo => :bar})).to eq(YAML.dump({:foo => :bar})) 155 | } 156 | 157 | it { 158 | expect(subject.serialize({:foo => 1})).to eq(YAML.dump({:foo => 1})) 159 | } 160 | 161 | it { 162 | expect(subject.serialize({:foo => "string"})).to eq(YAML.dump({:foo => "string"})) 163 | } 164 | 165 | it { 166 | expect(subject.serialize({:foo => MockModel.new})).to eq("---\n:foo: !ruby/object:MockModel\n model_id: 1\n model_type: TypeX\n") 167 | } 168 | end 169 | 170 | describe "Object" do 171 | it { 172 | expect(subject.serialize(:foo)).to eq(YAML.dump(:foo)) 173 | } 174 | 175 | it { 176 | expect(subject.serialize(1)).to eq(YAML.dump(1)) 177 | } 178 | 179 | it { 180 | expect(subject.serialize("string")).to eq(YAML.dump("string")) 181 | } 182 | 183 | it { 184 | expect(subject.serialize(MockModel.new)).to eq("--- !ruby/object:MockModel\nmodel_id: 1\nmodel_type: TypeX\n") 185 | } 186 | end 187 | end 188 | 189 | describe "#deserialize" do 190 | describe "Array" do 191 | it { 192 | expect(subject.deserialize(YAML.dump([]))).to eq([]) 193 | } 194 | 195 | it { 196 | expect(subject.deserialize(YAML.dump([1]))).to eq([1]) 197 | } 198 | 199 | it { 200 | expect(subject.deserialize(YAML.dump(["string"]))).to eq(["string"]) 201 | } 202 | 203 | it { 204 | expect_any_instance_of(MockModel).to receive(:find) 205 | subject.deserialize("---\n!ruby/object:MockModel\n model_id: 1\n model_type: TypeX\n") 206 | } 207 | end 208 | 209 | describe "Hash" do 210 | it { 211 | expect(subject.deserialize(YAML.dump({:foo => :bar}))).to eq({:foo => :bar}) 212 | } 213 | 214 | it { 215 | expect(subject.deserialize(YAML.dump({:foo => 1}))).to eq({:foo => 1}) 216 | } 217 | 218 | it { 219 | expect(subject.deserialize(YAML.dump({:foo => "string"}))).to eq({:foo => "string"}) 220 | } 221 | 222 | it { 223 | expect_any_instance_of(MockModel).to receive(:find) 224 | subject.deserialize("---\n:foo: !ruby/object:MockModel\n model_id: 1\n model_type: TypeX\n") 225 | } 226 | end 227 | 228 | describe "Object" do 229 | it { 230 | expect(subject.deserialize(YAML.dump(:foo))).to eq(:foo) 231 | } 232 | 233 | it { 234 | expect(subject.deserialize(YAML.dump(1))).to eq(1) 235 | } 236 | 237 | it { 238 | expect(subject.deserialize(YAML.dump("string"))).to eq("string") 239 | } 240 | 241 | it { 242 | expect_any_instance_of(MockModel).to receive(:find) 243 | subject.deserialize("---\n!ruby/object:MockModel\n model_id: 1\n model_type: TypeX\n") 244 | } 245 | end 246 | end 247 | end 248 | 249 | describe "unknown type helpers" do 250 | subject { Taskinator::Persistence::UnknownType } 251 | 252 | describe "#new" do 253 | it "instantiates new module instance" do 254 | instance = subject.new("foo") 255 | expect(instance).to_not be_nil 256 | expect(instance).to be_a(::Module) 257 | end 258 | 259 | it "yields same instance for same type" do 260 | instance1 = subject.new("foo") 261 | instance2 = subject.new("foo") 262 | expect(instance1).to eq(instance2) 263 | end 264 | end 265 | 266 | describe ".type" do 267 | it { 268 | instance = subject.new("foo") 269 | expect(instance.type).to eq("foo") 270 | } 271 | end 272 | 273 | describe ".to_s" do 274 | it { 275 | instance = subject.new("foo") 276 | expect(instance.to_s).to eq("Unknown type 'foo'.") 277 | } 278 | end 279 | 280 | describe ".allocate" do 281 | it "emulates Object#allocate" do 282 | instance = subject.new("foo") 283 | expect(instance.allocate).to eq(instance) 284 | end 285 | end 286 | 287 | describe ".accept" do 288 | it { 289 | instance = subject.new("foo") 290 | expect(instance).to respond_to(:accept) 291 | } 292 | end 293 | 294 | describe ".perform" do 295 | it "raises UnknownTypeError" do 296 | instance = subject.new("foo") 297 | expect { 298 | instance.perform(:foo, 1, false) 299 | }.to raise_error(Taskinator::Persistence::UnknownTypeError) 300 | end 301 | end 302 | 303 | describe "via executor" do 304 | it "raises UnknownTypeError" do 305 | instance = subject.new("foo") 306 | executor = Taskinator::Executor.new(instance) 307 | 308 | expect { 309 | executor.foo 310 | }.to raise_error(Taskinator::Persistence::UnknownTypeError) 311 | end 312 | end 313 | end 314 | 315 | describe "instance methods" do 316 | subject { 317 | klass = Class.new do 318 | include Taskinator::Persistence 319 | 320 | def self.base_key 321 | 'base_key' 322 | end 323 | 324 | attr_reader :uuid 325 | 326 | def initialize 327 | @uuid = Taskinator.generate_uuid 328 | end 329 | end 330 | klass.new 331 | } 332 | 333 | describe "#save" do 334 | pending 335 | end 336 | 337 | describe "#to_xml" do 338 | it { 339 | process = TestFlows::Task.create_process(1) 340 | expect(process.to_xml).to match(/xml/) 341 | } 342 | end 343 | 344 | describe "#key" do 345 | it { 346 | expect(subject.key).to match(/#{subject.uuid}/) 347 | } 348 | end 349 | 350 | describe "#process_uuid" do 351 | it { 352 | Taskinator.redis do |conn| 353 | conn.hset(subject.key, :process_uuid, subject.uuid) 354 | end 355 | 356 | expect(subject.process_uuid).to match(/#{subject.uuid}/) 357 | } 358 | end 359 | 360 | describe "#process_key" do 361 | it { 362 | Taskinator.redis do |conn| 363 | conn.hset(subject.key, :process_uuid, subject.uuid) 364 | end 365 | 366 | expect(subject.process_key).to match(/#{subject.uuid}/) 367 | } 368 | end 369 | 370 | describe "#load_workflow_state" do 371 | it { 372 | expect(subject.load_workflow_state).to eq(:initial) 373 | } 374 | end 375 | 376 | describe "#persist_workflow_state" do 377 | it { 378 | subject.persist_workflow_state(:active) 379 | expect(subject.load_workflow_state).to eq(:active) 380 | } 381 | end 382 | 383 | describe "#fail" do 384 | it "persists error information" do 385 | begin 386 | # raise this error in a block, so there is a backtrace! 387 | raise StandardError.new('a error') 388 | rescue => e 389 | subject.fail(e) 390 | end 391 | 392 | type, message, backtrace = Taskinator.redis do |conn| 393 | conn.hmget(subject.key, :error_type, :error_message, :error_backtrace) 394 | end 395 | 396 | expect(type).to eq('StandardError') 397 | expect(message).to eq('a error') 398 | expect(backtrace).to_not be_empty 399 | end 400 | end 401 | 402 | describe "#error" do 403 | it "retrieves error information" do 404 | error = nil 405 | begin 406 | # raise this error in a block, so there is a backtrace! 407 | raise StandardError.new('a error') 408 | rescue => e 409 | error = e 410 | subject.fail(error) 411 | end 412 | 413 | expect(subject.error).to eq([error.class.name, error.message, error.backtrace]) 414 | end 415 | end 416 | 417 | describe "#tasks_count" do 418 | it { 419 | Taskinator.redis do |conn| 420 | conn.hset(subject.process_key, :tasks_count, 99) 421 | end 422 | 423 | expect(subject.tasks_count).to eq(99) 424 | } 425 | end 426 | 427 | %w( 428 | failed 429 | cancelled 430 | completed 431 | ).each do |status| 432 | 433 | describe "#count_#{status}" do 434 | it { 435 | Taskinator.redis do |conn| 436 | conn.hset(subject.process_key, "tasks_#{status}", 99) 437 | end 438 | 439 | expect(subject.send(:"count_#{status}")).to eq(99) 440 | } 441 | end 442 | 443 | describe "#incr_#{status}" do 444 | it { 445 | Taskinator.redis do |conn| 446 | conn.hset(subject.process_key, "tasks_#{status}", 99) 447 | end 448 | 449 | subject.send(:"incr_#{status}") 450 | 451 | expect(subject.send(:"count_#{status}")).to eq(100) 452 | } 453 | end 454 | 455 | describe "#percentage_#{status}" do 456 | it { 457 | Taskinator.redis do |conn| 458 | conn.hmset( 459 | subject.process_key, 460 | [:tasks_count, 100], 461 | ["tasks_#{status}", 1] 462 | ) 463 | end 464 | 465 | expect(subject.send(:"percentage_#{status}")).to eq(1.0) 466 | } 467 | end 468 | 469 | end 470 | 471 | describe "#deincr_pending_tasks" do 472 | it { 473 | Taskinator.redis do |conn| 474 | conn.set("#{subject.key}.pending", 99) 475 | end 476 | 477 | pending = subject.deincr_pending_tasks 478 | 479 | expect(pending).to eq(98) 480 | } 481 | end 482 | 483 | describe "#process_options" do 484 | it { 485 | Taskinator.redis do |conn| 486 | conn.hset(subject.process_key, :options, YAML.dump({:foo => :bar})) 487 | end 488 | 489 | expect(subject.process_options).to eq(:foo => :bar) 490 | } 491 | end 492 | 493 | describe "#cleanup" do 494 | 495 | [ 496 | TestFlows::Task, 497 | TestFlows::Job, 498 | TestFlows::SubProcess, 499 | TestFlows::Sequential, 500 | TestFlows::Concurrent, 501 | TestFlows::EmptySequentialProcessTest, 502 | TestFlows::EmptyConcurrentProcessTest, 503 | TestFlows::NestedTask, 504 | ].each do |definition| 505 | 506 | describe "#{definition.name} expire immediately" do 507 | it { 508 | Taskinator.redis do |conn| 509 | # sanity check 510 | expect(conn.keys).to be_empty 511 | 512 | process = definition.create_process(1) 513 | 514 | # sanity check 515 | expect(conn.hget(process.key, :uuid)).to eq(process.uuid) 516 | 517 | process.cleanup(0) # immediately 518 | 519 | # ensure nothing left behind 520 | expect(conn.keys).to be_empty 521 | end 522 | } 523 | end 524 | 525 | end 526 | 527 | describe "expires in future" do 528 | it { 529 | Taskinator.redis do |conn| 530 | 531 | # sanity check 532 | expect(conn.keys).to be_empty 533 | 534 | process = TestFlows::Task.create_process(1) 535 | 536 | # sanity check 537 | expect(conn.hget(process.key, :uuid)).to eq(process.uuid) 538 | 539 | process.cleanup(2) 540 | 541 | # still available... 542 | expect(conn.hget(process.key, :uuid)).to_not be_nil 543 | recursively_enumerate_tasks(process.tasks) do |task| 544 | expect(conn.hget(task.key, :uuid)).to_not be_nil 545 | end 546 | 547 | sleep 3 548 | 549 | # ensure nothing left behind 550 | expect(conn.keys).to be_empty 551 | end 552 | } 553 | end 554 | 555 | end 556 | end 557 | end 558 | -------------------------------------------------------------------------------- /spec/taskinator/queues/active_job_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Queues::ActiveJobAdapter, :active_job do 4 | 5 | it_should_behave_like "a queue adapter", :active_job, Taskinator::Queues::ActiveJobAdapter 6 | 7 | let(:adapter) { Taskinator::Queues::ActiveJobAdapter } 8 | let(:uuid) { Taskinator.generate_uuid } 9 | 10 | subject { adapter.new } 11 | 12 | describe "CreateProcessWorker" do 13 | let(:args) { Taskinator::Persistence.serialize(:foo => :bar) } 14 | 15 | it "enqueues" do 16 | worker = adapter::CreateProcessWorker 17 | definition = MockDefinition.create 18 | subject.enqueue_create_process(definition, uuid, :foo => :bar) 19 | 20 | expect(worker).to have_been_enqueued.with(definition.name, uuid, args) 21 | end 22 | 23 | it "enqueues to specified queue" do 24 | worker = adapter::CreateProcessWorker 25 | definition = MockDefinition.create(:other) 26 | 27 | subject.enqueue_create_process(definition, uuid, :foo => :bar) 28 | 29 | expect(worker).to have_been_enqueued.with(definition.name, uuid, args).on_queue(:other) 30 | end 31 | 32 | it "calls worker" do 33 | expect_any_instance_of(Taskinator::CreateProcessWorker).to receive(:perform) 34 | adapter::CreateProcessWorker.new.perform(MockDefinition.create.name, uuid, args) 35 | end 36 | end 37 | 38 | describe "TaskWorker" do 39 | it "enqueues tasks" do 40 | worker = adapter::TaskWorker 41 | subject.enqueue_task(double('task', :uuid => uuid, :queue => nil)) 42 | 43 | expect(worker).to have_been_enqueued.with(uuid) 44 | end 45 | 46 | it "enqueues task to specified queue" do 47 | worker = adapter::TaskWorker 48 | subject.enqueue_task(double('task', :uuid => uuid, :queue => :other)) 49 | 50 | expect(worker).to have_been_enqueued.with(uuid).on_queue(:other) 51 | end 52 | 53 | it "calls task worker" do 54 | expect_any_instance_of(Taskinator::TaskWorker).to receive(:perform) 55 | adapter::TaskWorker.new.perform(uuid) 56 | end 57 | end 58 | 59 | end 60 | -------------------------------------------------------------------------------- /spec/taskinator/queues/delayed_job_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Queues::DelayedJobAdapter, :delayed_job do 4 | 5 | it_should_behave_like "a queue adapter", :delayed_job, Taskinator::Queues::DelayedJobAdapter 6 | 7 | let(:adapter) { Taskinator::Queues::DelayedJobAdapter } 8 | let(:uuid) { Taskinator.generate_uuid } 9 | 10 | subject { adapter.new } 11 | 12 | describe "CreateProcessWorker" do 13 | let(:args) { Taskinator::Persistence.serialize(:foo => :bar) } 14 | 15 | it "enqueues" do 16 | expect { 17 | subject.enqueue_create_process(MockDefinition.create, uuid, :foo => :bar) 18 | }.to change(Delayed::Job.queue, :size).by(1) 19 | end 20 | 21 | it "enqueues to specified queue" do 22 | definition = MockDefinition.create(:other) 23 | subject.enqueue_create_process(definition, uuid, :foo => :bar) 24 | expect(Delayed::Job.contains?(adapter::CreateProcessWorker, [definition.name, uuid, args], :other)).to be 25 | end 26 | 27 | it "calls worker" do 28 | expect_any_instance_of(Taskinator::CreateProcessWorker).to receive(:perform) 29 | adapter::CreateProcessWorker.new(MockDefinition.create.name, uuid, args).perform 30 | end 31 | end 32 | 33 | describe "TaskWorker" do 34 | it "enqueues tasks" do 35 | expect { 36 | subject.enqueue_task(double('task', :uuid => uuid, :queue => nil)) 37 | }.to change(Delayed::Job.queue, :size).by(1) 38 | end 39 | 40 | it "enqueues task to specified queue" do 41 | subject.enqueue_task(double('task', :uuid => uuid, :queue => :other)) 42 | expect(Delayed::Job.contains?(adapter::TaskWorker, uuid, :other)).to be 43 | end 44 | 45 | it "calls task worker" do 46 | expect_any_instance_of(Taskinator::TaskWorker).to receive(:perform) 47 | adapter::TaskWorker.new(uuid).perform 48 | end 49 | end 50 | 51 | end 52 | -------------------------------------------------------------------------------- /spec/taskinator/queues/resque_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Queues::ResqueAdapter, :resque do 4 | 5 | it_should_behave_like "a queue adapter", :resque, Taskinator::Queues::ResqueAdapter 6 | 7 | let(:adapter) { Taskinator::Queues::ResqueAdapter } 8 | let(:uuid) { Taskinator.generate_uuid } 9 | 10 | subject { adapter.new } 11 | 12 | describe "CreateProcessWorker" do 13 | let(:args) { Taskinator::Persistence.serialize(:foo => :bar) } 14 | 15 | it "enqueues" do 16 | worker = adapter::CreateProcessWorker 17 | definition = MockDefinition.create 18 | subject.enqueue_create_process(definition, uuid, :foo => :bar) 19 | 20 | expect(worker).to have_queued(definition.name, uuid, args) 21 | end 22 | 23 | it "enqueues to specified queue" do 24 | worker = adapter::CreateProcessWorker 25 | definition = MockDefinition.create(:other) 26 | subject.enqueue_create_process(definition, uuid, :foo => :bar) 27 | 28 | expect(worker).to have_queued(definition.name, uuid, args).in(:other) 29 | end 30 | 31 | it "calls worker" do 32 | expect_any_instance_of(Taskinator::CreateProcessWorker).to receive(:perform) 33 | adapter::CreateProcessWorker.perform(MockDefinition.create.name, uuid, args) 34 | end 35 | end 36 | 37 | describe "TaskWorker" do 38 | it "enqueues tasks" do 39 | worker = adapter::TaskWorker 40 | subject.enqueue_task(double('task', :uuid => uuid, :queue => nil)) 41 | 42 | expect(worker).to have_queued(uuid) 43 | end 44 | 45 | it "enqueues task to specified queue" do 46 | worker = adapter::TaskWorker 47 | subject.enqueue_task(double('task', :uuid => uuid, :queue => :other)) 48 | 49 | expect(worker).to have_queued(uuid).in(:other) 50 | end 51 | 52 | it "calls task worker" do 53 | expect_any_instance_of(Taskinator::TaskWorker).to receive(:perform) 54 | adapter::TaskWorker.perform(uuid) 55 | end 56 | end 57 | 58 | end 59 | -------------------------------------------------------------------------------- /spec/taskinator/queues/sidekiq_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Queues::SidekiqAdapter, :sidekiq do 4 | 5 | it_should_behave_like "a queue adapter", :sidekiq, Taskinator::Queues::SidekiqAdapter do 6 | let(:job) { double('job', :get_sidekiq_options => {}) } 7 | end 8 | 9 | let(:adapter) { Taskinator::Queues::SidekiqAdapter } 10 | let(:uuid) { Taskinator.generate_uuid } 11 | 12 | subject { adapter.new } 13 | 14 | describe "CreateProcessWorker" do 15 | let(:args) { Taskinator::Persistence.serialize(:foo => :bar) } 16 | 17 | it "enqueues" do 18 | worker = adapter::CreateProcessWorker 19 | definition = MockDefinition.create 20 | subject.enqueue_create_process(definition, uuid, :foo => :bar) 21 | expect(worker).to have_enqueued_sidekiq_job(definition.name, uuid, args) 22 | end 23 | 24 | it "enqueues to specified queue" do 25 | subject.enqueue_create_process(MockDefinition.create(:other), uuid, :foo => :bar) 26 | expect(adapter::CreateProcessWorker).to be_processed_in_x(:other) 27 | end 28 | 29 | it "calls worker" do 30 | definition = MockDefinition.create 31 | expect_any_instance_of(Taskinator::CreateProcessWorker).to receive(:perform) 32 | adapter::CreateProcessWorker.new.perform(definition.name, uuid, args) 33 | end 34 | end 35 | 36 | describe "TaskWorker" do 37 | it "enqueues tasks" do 38 | worker = adapter::TaskWorker 39 | task = double('task', :uuid => uuid, :queue => nil) 40 | subject.enqueue_task(task) 41 | expect(worker).to have_enqueued_sidekiq_job(task.uuid) 42 | end 43 | 44 | it "enqueues task to specified queue" do 45 | subject.enqueue_task(double('task', :uuid => uuid, :queue => :other)) 46 | expect(adapter::TaskWorker).to be_processed_in_x(:other) 47 | end 48 | 49 | it "calls task worker" do 50 | expect_any_instance_of(Taskinator::TaskWorker).to receive(:perform) 51 | adapter::TaskWorker.new.perform(uuid) 52 | end 53 | end 54 | 55 | end 56 | -------------------------------------------------------------------------------- /spec/taskinator/queues/test_queue_adapter_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Queues::TestQueueAdapter do 4 | 5 | # sanity check for the test adapter 6 | 7 | it_should_behave_like "a queue adapter", :test_queue, Taskinator::Queues::TestQueueAdapter 8 | 9 | end 10 | -------------------------------------------------------------------------------- /spec/taskinator/queues_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Queues do 4 | 5 | it "raise error for an unknown adapter" do 6 | expect { 7 | Taskinator::Queues.create_adapter(:unknown) 8 | }.to raise_error(StandardError) 9 | end 10 | 11 | it "passes configuration to adapter initializer" do 12 | config = {:a => :b, :c => :d} 13 | expect(Taskinator::Queues).to receive(:create_test_adapter).with(config) 14 | 15 | Taskinator::Queues.create_adapter(:test, config) 16 | end 17 | 18 | end 19 | -------------------------------------------------------------------------------- /spec/taskinator/task_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Task do 4 | 5 | let(:definition) { TestDefinition } 6 | 7 | let(:process) do 8 | Class.new(Taskinator::Process) do 9 | include ProcessMethods 10 | end.new(definition) 11 | end 12 | 13 | describe "Base" do 14 | 15 | subject do 16 | Class.new(Taskinator::Task) do 17 | include TaskMethods 18 | end.new(process) 19 | end 20 | 21 | describe "#initialize" do 22 | it { expect(subject.process).to_not be_nil } 23 | it { expect(subject.process).to eq(process) } 24 | it { expect(subject.uuid).to_not be_nil } 25 | it { expect(subject.options).to_not be_nil } 26 | end 27 | 28 | describe "#<==>" do 29 | it { expect(subject).to be_a(::Comparable) } 30 | it { 31 | uuid = subject.uuid 32 | expect(subject == double('test', :uuid => uuid)).to be 33 | } 34 | 35 | it { 36 | expect(subject == double('test', :uuid => 'xxx')).to_not be 37 | } 38 | end 39 | 40 | describe "#to_s" do 41 | it { expect(subject.to_s).to match(/#{subject.uuid}/) } 42 | end 43 | 44 | describe "#queue" do 45 | it { 46 | expect(subject.queue).to be_nil 47 | } 48 | 49 | it { 50 | task = Class.new(Taskinator::Task).new(process, :queue => :foo) 51 | expect(task.queue).to eq(:foo) 52 | } 53 | end 54 | 55 | describe "#current_state" do 56 | it { expect(subject).to be_a(Taskinator::Workflow) } 57 | it { expect(subject.current_state).to_not be_nil } 58 | it { expect(subject.current_state).to eq(:initial) } 59 | end 60 | 61 | describe "workflow" do 62 | describe "#enqueue!" do 63 | it { expect(subject).to respond_to(:enqueue!) } 64 | it { 65 | expect(subject).to receive(:enqueue) 66 | subject.enqueue! 67 | } 68 | it { 69 | subject.enqueue! 70 | expect(subject.current_state).to eq(:enqueued) 71 | } 72 | end 73 | 74 | describe "#start!" do 75 | it { expect(subject).to respond_to(:start!) } 76 | it { 77 | expect(subject).to receive(:start) 78 | subject.start! 79 | } 80 | it { 81 | subject.start! 82 | expect(subject.current_state).to eq(:processing) 83 | } 84 | end 85 | 86 | describe "#complete!" do 87 | it { expect(subject).to respond_to(:complete!) } 88 | it { 89 | expect(subject).to receive(:complete) 90 | subject.start! 91 | subject.complete! 92 | expect(subject.current_state).to eq(:completed) 93 | } 94 | end 95 | 96 | describe "#cancel!" do 97 | it { expect(subject).to respond_to(:cancel!) } 98 | it { 99 | expect(subject).to receive(:cancel) 100 | subject.start! 101 | subject.cancel! 102 | expect(subject.current_state).to eq(:cancelled) 103 | } 104 | end 105 | 106 | describe "#fail!" do 107 | it { expect(subject).to respond_to(:fail!) } 108 | it { 109 | error = StandardError.new 110 | expect(subject).to receive(:fail).with(error) 111 | expect(process).to receive(:task_failed).with(subject, error) 112 | subject.start! 113 | subject.fail!(error) 114 | } 115 | it { 116 | subject.start! 117 | subject.fail!(StandardError.new) 118 | expect(subject.current_state).to eq(:failed) 119 | } 120 | end 121 | 122 | describe "#paused?" do 123 | it { expect(subject.paused?).to_not be } 124 | it { 125 | process.start! 126 | process.pause! 127 | expect(subject.paused?).to eq(true) 128 | } 129 | end 130 | 131 | describe "#cancelled?" do 132 | it { expect(subject.cancelled?).to_not be } 133 | it { 134 | process.cancel! 135 | expect(subject.cancelled?).to be 136 | } 137 | end 138 | end 139 | 140 | describe "#next" do 141 | it { expect(subject).to respond_to(:next) } 142 | it { expect(subject).to respond_to(:next=) } 143 | end 144 | 145 | describe "#accept" do 146 | it { expect(subject).to be_a(Taskinator::Persistence) } 147 | 148 | it { 149 | expect(subject).to receive(:accept) 150 | subject.save 151 | } 152 | 153 | it { 154 | visitor = double('visitor') 155 | expect(visitor).to receive(:visit_attribute).with(:uuid) 156 | expect(visitor).to receive(:visit_process_reference).with(:process) 157 | expect(visitor).to receive(:visit_type).with(:definition) 158 | expect(visitor).to receive(:visit_task_reference).with(:next) 159 | expect(visitor).to receive(:visit_args).with(:options) 160 | expect(visitor).to receive(:visit_attribute).with(:queue) 161 | expect(visitor).to receive(:visit_attribute_time).with(:created_at) 162 | expect(visitor).to receive(:visit_attribute_time).with(:updated_at) 163 | 164 | subject.accept(visitor) 165 | } 166 | end 167 | 168 | describe "#tasks_count" do 169 | it { 170 | process_uuid = SecureRandom.hex 171 | allow(subject).to receive(:process_uuid) { process_uuid } 172 | expect(subject.tasks_count).to eq(0) 173 | } 174 | end 175 | end 176 | 177 | describe Taskinator::Task::Step do 178 | 179 | subject { Taskinator::Task.define_step_task(process, :do_task, {:a => 1, :b => 2}) } 180 | 181 | it_should_behave_like "a task", Taskinator::Task::Step 182 | 183 | describe ".define_step_task" do 184 | it "sets the queue to use" do 185 | task = Taskinator::Task.define_step_task(process, :do_task, {:a => 1, :b => 2}, :queue => :foo) 186 | expect(task.queue).to eq(:foo) 187 | end 188 | end 189 | 190 | describe "#executor" do 191 | it { expect(subject.executor).to_not be_nil } 192 | it { expect(subject.executor).to be_a(definition) } 193 | 194 | it "handles failure" do 195 | error = StandardError.new 196 | allow(subject.executor).to receive(subject.method).with(*subject.args).and_raise(error) 197 | expect(subject).to receive(:fail!).with(error) 198 | expect { 199 | subject.start! 200 | }.to raise_error(error) 201 | end 202 | end 203 | 204 | describe "#enqueue!" do 205 | it { 206 | expect { 207 | subject.enqueue! 208 | }.to change { Taskinator.queue.tasks.length }.by(1) 209 | } 210 | 211 | it "is instrumented" do 212 | allow(subject.executor).to receive(subject.method).with(*subject.args) 213 | 214 | instrumentation_block = SpecSupport::Block.new 215 | 216 | expect(instrumentation_block).to receive(:call) do |*args| 217 | expect(args.first).to eq('taskinator.task.enqueued') 218 | end 219 | 220 | TestInstrumenter.subscribe(instrumentation_block, /taskinator.task/) do 221 | subject.enqueue! 222 | end 223 | end 224 | end 225 | 226 | describe "#start!" do 227 | before do 228 | allow(process).to receive(:task_completed).with(subject) 229 | end 230 | 231 | it "invokes executor" do 232 | expect(subject.executor).to receive(subject.method).with(*subject.args) 233 | subject.start! 234 | end 235 | 236 | it "provides execution context" do 237 | executor = Taskinator::Executor.new(definition, subject) 238 | 239 | method = subject.method 240 | 241 | executor.singleton_class.class_eval do 242 | define_method method do |*args| 243 | # this method executes in the scope of the executor 244 | # store the context in an instance variable 245 | @exec_context = self 246 | end 247 | end 248 | 249 | # replace the internal executor instance for the task 250 | # with this one, so we can hook into the methods 251 | subject.instance_eval { @executor = executor } 252 | 253 | # task start will invoke the method on the executor 254 | subject.start! 255 | 256 | # extract the instance variable 257 | exec_context = executor.instance_eval { @exec_context } 258 | 259 | expect(exec_context).to eq(executor) 260 | expect(exec_context.uuid).to eq(subject.uuid) 261 | expect(exec_context.options).to eq(subject.options) 262 | end 263 | 264 | it "throws an exception for unknown definition type" do 265 | executor = Taskinator::Executor.new(Taskinator::Persistence::UnknownType.new("Foo"), subject) 266 | 267 | # replace the internal executor instance for the task 268 | # with this one, so we can hook into the methods 269 | subject.instance_eval { @executor = executor } 270 | 271 | expect { 272 | subject.start! 273 | }.to raise_error(Taskinator::Persistence::UnknownTypeError) 274 | end 275 | 276 | it "is instrumented" do 277 | instrumentation_block = SpecSupport::Block.new 278 | 279 | expect(instrumentation_block).to receive(:call) do |*args| 280 | expect(args.first).to eq('taskinator.task.processing') 281 | end 282 | 283 | expect(instrumentation_block).to receive(:call) do |*args| 284 | expect(args.first).to eq('taskinator.task.completed') 285 | end 286 | 287 | TestInstrumenter.subscribe(instrumentation_block) do 288 | subject.start! 289 | end 290 | end 291 | end 292 | 293 | describe "#complete" do 294 | it "notifies parent process" do 295 | expect(process).to receive(:task_completed).with(subject) 296 | 297 | subject.complete! 298 | end 299 | 300 | it "is instrumented" do 301 | allow(process).to receive(:task_completed).with(subject) 302 | 303 | instrumentation_block = SpecSupport::Block.new 304 | 305 | expect(instrumentation_block).to receive(:call) do |*args| 306 | expect(args.first).to eq('taskinator.task.completed') 307 | end 308 | 309 | TestInstrumenter.subscribe(instrumentation_block, /taskinator.task/) do 310 | subject.complete! 311 | end 312 | end 313 | end 314 | 315 | describe "#accept" do 316 | it { 317 | expect(subject).to receive(:accept) 318 | subject.save 319 | } 320 | 321 | it { 322 | visitor = double('visitor') 323 | expect(visitor).to receive(:visit_type).with(:definition) 324 | expect(visitor).to receive(:visit_attribute).with(:uuid) 325 | expect(visitor).to receive(:visit_process_reference).with(:process) 326 | expect(visitor).to receive(:visit_task_reference).with(:next) 327 | expect(visitor).to receive(:visit_args).with(:options) 328 | expect(visitor).to receive(:visit_attribute).with(:method) 329 | expect(visitor).to receive(:visit_args).with(:args) 330 | expect(visitor).to receive(:visit_attribute).with(:queue) 331 | expect(visitor).to receive(:visit_attribute_time).with(:created_at) 332 | expect(visitor).to receive(:visit_attribute_time).with(:updated_at) 333 | 334 | subject.accept(visitor) 335 | } 336 | end 337 | 338 | describe "#inspect" do 339 | it { expect(subject.inspect).to_not be_nil } 340 | it { expect(subject.inspect).to include(definition.name) } 341 | end 342 | end 343 | 344 | describe Taskinator::Task::Job do 345 | 346 | subject { Taskinator::Task.define_job_task(process, TestJob, [1, {:a => 1, :b => 2}]) } 347 | 348 | it_should_behave_like "a task", Taskinator::Task::Job 349 | 350 | describe ".define_job_task" do 351 | it "sets the queue to use" do 352 | task = Taskinator::Task.define_job_task(process, TestJob, [1, {:a => 1, :b => 2}], :queue => :foo) 353 | expect(task.queue).to eq(:foo) 354 | end 355 | end 356 | 357 | describe "#enqueue!" do 358 | it { 359 | expect { 360 | subject.enqueue! 361 | }.to change { Taskinator.queue.tasks.length }.by(1) 362 | } 363 | 364 | it "is instrumented" do 365 | instrumentation_block = SpecSupport::Block.new 366 | 367 | expect(instrumentation_block).to receive(:call) do |*args| 368 | expect(args.first).to eq('taskinator.task.enqueued') 369 | end 370 | 371 | TestInstrumenter.subscribe(instrumentation_block, /taskinator.task/) do 372 | subject.enqueue! 373 | end 374 | end 375 | end 376 | 377 | describe "#start" do 378 | it { 379 | task = Taskinator::Task.define_job_task(process, TestJobClass, [1, {:a => 1, :b => 2}]) 380 | expect(process).to receive(:task_completed).with(task) 381 | expect_any_instance_of(TestJobClass).to receive(:perform).with(1, {:a => 1, :b => 2}) 382 | task.start! 383 | } 384 | 385 | it { 386 | task = Taskinator::Task.define_job_task(process, TestJobModule, [2, {:a => 1, :b => 2}]) 387 | expect(process).to receive(:task_completed).with(task) 388 | expect(TestJobModule).to receive(:perform).with(2, {:a => 1, :b => 2}) 389 | task.start! 390 | } 391 | 392 | it { 393 | task = Taskinator::Task.define_job_task(process, TestJobClassNoArgs, nil) 394 | expect(process).to receive(:task_completed).with(task) 395 | expect_any_instance_of(TestJobClassNoArgs).to receive(:perform).and_call_original 396 | task.start! 397 | } 398 | 399 | it { 400 | task = Taskinator::Task.define_job_task(process, TestJobModuleNoArgs, nil) 401 | expect(process).to receive(:task_completed).with(task) 402 | expect(TestJobModuleNoArgs).to receive(:perform).and_call_original 403 | task.start! 404 | } 405 | 406 | it "throws an exception when unknown job type" do 407 | task = Taskinator::Task.define_job_task(process, Taskinator::Persistence::UnknownType.new("Foo"), nil) 408 | 409 | expect { 410 | task.start! 411 | }.to raise_error(Taskinator::Persistence::UnknownTypeError) 412 | end 413 | 414 | it "handles failure" do 415 | task = Taskinator::Task.define_job_task(process, TestJobError, nil) 416 | 417 | expect { 418 | task.start! 419 | }.to raise_error(ArgumentError) 420 | end 421 | 422 | it "is instrumented" do 423 | allow(process).to receive(:task_completed).with(subject) 424 | 425 | allow(TestJob).to receive(:perform).with(1, {:a => 1, :b => 2}) 426 | 427 | instrumentation_block = SpecSupport::Block.new 428 | 429 | expect(instrumentation_block).to receive(:call) do |*args| 430 | expect(args.first).to eq('taskinator.task.processing') 431 | end 432 | 433 | # special case, since when the method returns, the task is considered to be complete 434 | expect(instrumentation_block).to receive(:call) do |*args| 435 | expect(args.first).to eq('taskinator.task.completed') 436 | end 437 | 438 | TestInstrumenter.subscribe(instrumentation_block, /taskinator.task/) do 439 | subject.start! 440 | end 441 | end 442 | end 443 | 444 | describe "#complete" do 445 | it "notifies parent process" do 446 | expect(process).to receive(:task_completed).with(subject) 447 | 448 | subject.complete! 449 | end 450 | 451 | it "is instrumented" do 452 | allow(process).to receive(:task_completed).with(subject) 453 | 454 | instrumentation_block = SpecSupport::Block.new 455 | 456 | expect(instrumentation_block).to receive(:call) do |*args| 457 | expect(args.first).to eq('taskinator.task.completed') 458 | end 459 | 460 | TestInstrumenter.subscribe(instrumentation_block, /taskinator.task/) do 461 | subject.complete! 462 | end 463 | end 464 | end 465 | 466 | describe "#accept" do 467 | it { 468 | expect(subject).to receive(:accept) 469 | subject.save 470 | } 471 | 472 | it { 473 | visitor = double('visitor') 474 | expect(visitor).to receive(:visit_type).with(:definition) 475 | expect(visitor).to receive(:visit_attribute).with(:uuid) 476 | expect(visitor).to receive(:visit_process_reference).with(:process) 477 | expect(visitor).to receive(:visit_task_reference).with(:next) 478 | expect(visitor).to receive(:visit_args).with(:options) 479 | expect(visitor).to receive(:visit_type).with(:job) 480 | expect(visitor).to receive(:visit_args).with(:args) 481 | expect(visitor).to receive(:visit_attribute).with(:queue) 482 | expect(visitor).to receive(:visit_attribute_time).with(:created_at) 483 | expect(visitor).to receive(:visit_attribute_time).with(:updated_at) 484 | 485 | subject.accept(visitor) 486 | } 487 | end 488 | 489 | describe "#inspect" do 490 | it { expect(subject.inspect).to_not be_nil } 491 | it { expect(subject.inspect).to include(definition.name) } 492 | end 493 | end 494 | 495 | describe Taskinator::Task::SubProcess do 496 | 497 | let(:sub_process) do 498 | Class.new(Taskinator::Process) do 499 | include ProcessMethods 500 | end.new(definition) 501 | end 502 | 503 | subject { Taskinator::Task.define_sub_process_task(process, sub_process) } 504 | 505 | it_should_behave_like "a task", Taskinator::Task::SubProcess 506 | 507 | describe ".define_sub_process_task" do 508 | it "sets the queue to use" do 509 | task = Taskinator::Task.define_sub_process_task(process, sub_process, :queue => :foo) 510 | expect(task.queue).to eq(:foo) 511 | end 512 | end 513 | 514 | describe "#enqueue!" do 515 | context "without tasks" do 516 | it { 517 | expect { 518 | subject.enqueue! 519 | }.to change { Taskinator.queue.tasks.length }.by(0) 520 | } 521 | end 522 | 523 | it "delegates to sub process" do 524 | expect(sub_process).to receive(:enqueue!) 525 | subject.enqueue! 526 | end 527 | 528 | it "is instrumented" do 529 | instrumentation_block = SpecSupport::Block.new 530 | 531 | expect(instrumentation_block).to receive(:call) do |*args| 532 | expect(args.first).to eq('taskinator.task.enqueued') 533 | end 534 | 535 | TestInstrumenter.subscribe(instrumentation_block, /taskinator.task/) do 536 | subject.enqueue! 537 | end 538 | end 539 | end 540 | 541 | describe "#start!" do 542 | it "delegates to sub process" do 543 | expect(sub_process).to receive(:start) 544 | subject.start! 545 | end 546 | 547 | it "handles failure" do 548 | error = StandardError.new 549 | allow(sub_process).to receive(:start!).and_raise(error) 550 | expect(subject).to receive(:fail!).with(error) 551 | expect { 552 | subject.start! 553 | }.to raise_error(error) 554 | end 555 | 556 | it "is instrumented" do 557 | instrumentation_block = SpecSupport::Block.new 558 | 559 | expect(instrumentation_block).to receive(:call) do |*args| 560 | expect(args.first).to eq('taskinator.task.processing') 561 | end 562 | 563 | TestInstrumenter.subscribe(instrumentation_block, /taskinator.task/) do 564 | subject.start! 565 | end 566 | end 567 | end 568 | 569 | describe "#complete" do 570 | it "notifies parent process" do 571 | expect(process).to receive(:task_completed).with(subject) 572 | 573 | subject.complete! 574 | end 575 | 576 | it "is instrumented" do 577 | allow(process).to receive(:task_completed).with(subject) 578 | 579 | instrumentation_block = SpecSupport::Block.new 580 | 581 | expect(instrumentation_block).to receive(:call) do |*args| 582 | expect(args.first).to eq('taskinator.task.completed') 583 | end 584 | 585 | TestInstrumenter.subscribe(instrumentation_block, /taskinator.task/) do 586 | subject.complete! 587 | end 588 | end 589 | end 590 | 591 | describe "#accept" do 592 | it { 593 | expect(subject).to receive(:accept) 594 | subject.save 595 | } 596 | 597 | it { 598 | visitor = double('visitor') 599 | expect(visitor).to receive(:visit_attribute).with(:uuid) 600 | expect(visitor).to receive(:visit_process_reference).with(:process) 601 | expect(visitor).to receive(:visit_type).with(:definition) 602 | expect(visitor).to receive(:visit_task_reference).with(:next) 603 | expect(visitor).to receive(:visit_args).with(:options) 604 | expect(visitor).to receive(:visit_process).with(:sub_process) 605 | expect(visitor).to receive(:visit_attribute).with(:queue) 606 | expect(visitor).to receive(:visit_attribute_time).with(:created_at) 607 | expect(visitor).to receive(:visit_attribute_time).with(:updated_at) 608 | 609 | subject.accept(visitor) 610 | } 611 | end 612 | 613 | describe "#inspect" do 614 | it { expect(subject.inspect).to_not be_nil } 615 | it { expect(subject.inspect).to include(definition.name) } 616 | end 617 | end 618 | 619 | end 620 | -------------------------------------------------------------------------------- /spec/taskinator/task_worker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::TaskWorker do 4 | 5 | def mock_task(paused=false, cancelled=false, can_complete=false) 6 | double('task', :paused? => paused, :cancelled? => cancelled, :can_complete? => can_complete) 7 | end 8 | 9 | let(:uuid) { Taskinator.generate_uuid } 10 | 11 | subject { Taskinator::TaskWorker.new(uuid) } 12 | 13 | it "should fetch the task" do 14 | task = mock_task 15 | expect(Taskinator::Task).to receive(:fetch).with(uuid) { task } 16 | allow(task).to receive(:start!) 17 | subject.perform 18 | end 19 | 20 | it "should start the task" do 21 | task = mock_task 22 | allow(Taskinator::Task).to receive(:fetch).with(uuid) { task } 23 | expect(task).to receive(:start!) 24 | subject.perform 25 | end 26 | 27 | it "should not start if paused" do 28 | task = mock_task(true, false) 29 | allow(Taskinator::Task).to receive(:fetch).with(uuid) { task } 30 | expect(task).to_not receive(:start!) 31 | subject.perform 32 | end 33 | 34 | it "should not start if cancelled" do 35 | task = mock_task(false, true) 36 | allow(Taskinator::Task).to receive(:fetch).with(uuid) { task } 37 | expect(task).to_not receive(:start!) 38 | subject.perform 39 | end 40 | 41 | it "should fail if task raises an error" do 42 | task = mock_task 43 | allow(Taskinator::Task).to receive(:fetch).with(uuid) { task } 44 | allow(task).to receive(:start!) { raise StandardError } 45 | expect { 46 | subject.perform 47 | }.to raise_error(StandardError) 48 | end 49 | 50 | end 51 | -------------------------------------------------------------------------------- /spec/taskinator/taskinator_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator do 4 | subject { Taskinator } 5 | 6 | describe "#options" do 7 | it { expect(subject.options).to be_a(Hash) } 8 | it { 9 | options = { :a => 1, :b => 2 } 10 | subject.options = options 11 | expect(subject.options).to eq(options) 12 | } 13 | end 14 | 15 | describe "#configure" do 16 | it "yields to block" do 17 | block = SpecSupport::Block.new 18 | expect(block).to receive(:call).with(subject) 19 | subject.configure(&block) 20 | end 21 | end 22 | 23 | describe "#redis" do 24 | it "yields to block" do 25 | block = SpecSupport::Block.new 26 | expect(block).to receive(:call) 27 | subject.redis(&block) 28 | end 29 | 30 | it "raise error when no block" do 31 | expect { 32 | subject.redis 33 | }.to raise_error(ArgumentError) 34 | end 35 | end 36 | 37 | describe "#redis_pool" do 38 | it { expect(subject.redis_pool).to_not be_nil } 39 | end 40 | 41 | describe "#queue_config" do 42 | it { 43 | subject.queue_config = {:a => 1} 44 | expect(subject.queue_config).to eq({:a => 1}) 45 | } 46 | end 47 | 48 | describe "#logger" do 49 | it { expect(subject.logger).to_not be_nil } 50 | it { 51 | logger = Logger.new(File::NULL) 52 | subject.logger = logger 53 | expect(subject.logger).to eq(logger) 54 | subject.logger = nil 55 | } 56 | end 57 | 58 | describe "#instrumenter" do 59 | it { expect(subject.instrumenter).to_not be_nil } 60 | 61 | it { 62 | orig_instrumenter = subject.instrumenter 63 | 64 | instrumenter = Class.new().new 65 | subject.instrumenter = instrumenter 66 | expect(subject.instrumenter).to eq(instrumenter) 67 | 68 | subject.instrumenter = orig_instrumenter 69 | } 70 | 71 | it "yields to given block" do 72 | block = SpecSupport::Block.new 73 | expect(block).to receive(:call) 74 | 75 | subject.instrumenter.instrument(:foo, :bar => :baz, &block) 76 | end 77 | 78 | it "instruments event, when activesupport is referenced" do 79 | block = SpecSupport::Block.new 80 | expect(block).to receive(:call) 81 | 82 | # temporary subscription 83 | ActiveSupport::Notifications.subscribed(block, /.*/) do 84 | subject.instrumenter.instrument(:foo, :bar) do 85 | :baz 86 | end 87 | end 88 | end 89 | end 90 | 91 | [ 92 | Taskinator::NoOpInstrumenter, 93 | Taskinator::ConsoleInstrumenter 94 | ].each do |instrumenter| 95 | describe instrumenter do 96 | it "yields to given block" do 97 | instance = instrumenter.new 98 | 99 | block = SpecSupport::Block.new 100 | expect(block).to receive(:call) 101 | 102 | instance.instrument(:foo, :bar => :baz, &block) 103 | end 104 | end 105 | end 106 | 107 | end 108 | -------------------------------------------------------------------------------- /spec/taskinator/tasks_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Tasks do 4 | 5 | class Element 6 | attr_accessor :next 7 | end 8 | 9 | it { expect(subject).to be_a(::Enumerable) } 10 | 11 | it { expect(subject).to respond_to(:add) } 12 | it { expect(subject).to respond_to(:<<) } 13 | it { expect(subject).to respond_to(:push) } 14 | it { expect(subject).to respond_to(:each) } 15 | it { expect(subject).to respond_to(:empty?) } 16 | it { expect(subject).to respond_to(:head) } 17 | 18 | describe "#initialize" do 19 | it "starts with nil head" do 20 | instance = Taskinator::Tasks.new 21 | expect(instance.head).to be_nil 22 | end 23 | 24 | it "assigns head to first element" do 25 | first = double 26 | instance = Taskinator::Tasks.new(first) 27 | expect(instance.head).to eq(first) 28 | end 29 | end 30 | 31 | describe "#add" do 32 | it "assigns to head for first element" do 33 | first = Element.new 34 | instance = Taskinator::Tasks.new 35 | instance.add(first) 36 | expect(instance.head).to eq(first) 37 | end 38 | 39 | it "links first element to the second element" do 40 | first = Element.new 41 | second = Element.new 42 | 43 | expect(first).to receive(:next=).with(second) 44 | 45 | instance = Taskinator::Tasks.new(first) 46 | instance.add(second) 47 | end 48 | 49 | it "links second element to the third element" do 50 | first = Element.new 51 | second = Element.new 52 | third = Element.new 53 | 54 | expect(second).to receive(:next=).with(third) 55 | 56 | instance = Taskinator::Tasks.new(first) 57 | instance.add(second) 58 | instance.add(third) 59 | end 60 | end 61 | 62 | describe "#each" do 63 | it "yields enumerator if no block given" do 64 | block = SpecSupport::Block.new 65 | expect(block).to receive(:call).exactly(3).times 66 | 67 | instance = Taskinator::Tasks.new 68 | 3.times { instance.add(Element.new) } 69 | 70 | enumerator = instance.each 71 | 72 | enumerator.each(&block) 73 | end 74 | 75 | it "enumerates elements" do 76 | block = SpecSupport::Block.new 77 | expect(block).to receive(:call).exactly(3).times 78 | 79 | instance = Taskinator::Tasks.new 80 | 3.times { instance.add(Element.new) } 81 | 82 | instance.each(&block) 83 | end 84 | 85 | it "does not enumerate when empty" do 86 | block = SpecSupport::Block.new 87 | expect(block).to_not receive(:call) 88 | 89 | instance = Taskinator::Tasks.new 90 | 91 | instance.each(&block) 92 | end 93 | end 94 | 95 | describe "#empty?" do 96 | it { expect(Taskinator::Tasks.new.empty?).to be } 97 | it { expect(Taskinator::Tasks.new(Element.new).empty?).to_not be } 98 | end 99 | 100 | end 101 | -------------------------------------------------------------------------------- /spec/taskinator/test_flows_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe TestFlows do 4 | 5 | [ 6 | TestFlows::Task, 7 | TestFlows::Job, 8 | TestFlows::SubProcess, 9 | TestFlows::Sequential 10 | ].each do |definition| 11 | 12 | describe definition.name do 13 | 14 | it "should have one task" do 15 | process = definition.create_process(1) 16 | expect(process.tasks_count).to eq(1) 17 | end 18 | 19 | it "should have 3 tasks" do 20 | process = definition.create_process(3) 21 | expect(process.tasks_count).to eq(3) 22 | end 23 | 24 | %w( 25 | failed 26 | cancelled 27 | completed 28 | ).each do |status| 29 | 30 | describe "count_#{status}" do 31 | it { 32 | process = definition.create_process(1) 33 | expect(process.send(:"count_#{status}")).to eq(0) 34 | } 35 | 36 | it { 37 | process = definition.create_process(2) 38 | expect(process.send(:"count_#{status}")).to eq(0) 39 | } 40 | end 41 | 42 | describe "incr_#{status}" do 43 | it { 44 | process = definition.create_process(1) 45 | process.send(:"incr_#{status}") 46 | expect(process.send(:"count_#{status}")).to eq(1) 47 | } 48 | 49 | it { 50 | process = definition.create_process(4) 51 | 4.times do |i| 52 | process.send(:"incr_#{status}") 53 | expect(process.send(:"count_#{status}")).to eq(i + 1) 54 | end 55 | } 56 | 57 | it "should increment completed count" do 58 | process = definition.create_process(10) 59 | recursively_enumerate_tasks(process.tasks) do |task| 60 | task.send(:"incr_#{status}") 61 | end 62 | expect(process.send(:"count_#{status}")).to eq(10) 63 | end 64 | end 65 | 66 | describe "percentage_#{status}" do 67 | it { 68 | process = definition.create_process(1) 69 | expect(process.send(:"percentage_#{status}")).to eq(0.0) 70 | } 71 | 72 | it { 73 | process = definition.create_process(4) 74 | expect(process.send(:"percentage_#{status}")).to eq(0.0) 75 | 76 | count = 4 77 | count.times do |i| 78 | process.send(:"incr_#{status}") 79 | expect(process.send(:"percentage_#{status}")).to eq( ((i + 1.0) / count) * 100.0 ) 80 | end 81 | } 82 | end 83 | 84 | end 85 | end 86 | end 87 | 88 | describe "scenarios" do 89 | 90 | before do 91 | # use the "synchronous" queue 92 | Taskinator.queue_adapter = :test_queue_worker 93 | end 94 | 95 | context "empty subprocesses" do 96 | 97 | context "sequential" do 98 | let(:definition) { TestFlows::EmptySequentialProcessTest } 99 | subject { definition.create_process } 100 | 101 | it "contains 3 tasks" do 102 | expect(subject.tasks.length).to eq(3) 103 | end 104 | 105 | it "invokes each task" do 106 | expect_any_instance_of(definition).to receive(:task_0) 107 | expect_any_instance_of(definition).to receive(:task_1) 108 | expect_any_instance_of(definition).to receive(:task_2) 109 | 110 | expect { 111 | subject.enqueue! 112 | }.to change { Taskinator.queue.tasks.length }.by(3) 113 | end 114 | end 115 | 116 | context "concurrent" do 117 | let(:definition) { TestFlows::EmptyConcurrentProcessTest } 118 | subject { definition.create_process } 119 | 120 | it "contains 3 tasks" do 121 | expect(subject.tasks.length).to eq(3) 122 | end 123 | 124 | it "invokes each task" do 125 | expect_any_instance_of(definition).to receive(:task_0) 126 | expect_any_instance_of(definition).to receive(:task_1) 127 | expect_any_instance_of(definition).to receive(:task_2) 128 | 129 | expect { 130 | subject.enqueue! 131 | }.to change { Taskinator.queue.tasks.length }.by(3) 132 | end 133 | end 134 | 135 | end 136 | end 137 | 138 | describe "statuses" do 139 | describe "task" do 140 | before do 141 | # override enqueue 142 | allow_any_instance_of(Taskinator::Task::Step).to receive(:enqueue!) { |task| 143 | # emulate the worker starting the task 144 | task.start! 145 | } 146 | end 147 | 148 | let(:task_count) { 2 } 149 | let(:definition) { TestFlows::Task } 150 | subject { definition.create_process(task_count) } 151 | 152 | it "reports process and task state" do 153 | 154 | instrumenter = TestInstrumenter.new do |name, payload| 155 | 156 | case name 157 | when 'taskinator.process.created', 'taskinator.process.saved' 158 | expect(payload[:state]).to eq(:initial) 159 | when 'taskinator.process.processing' 160 | expect(payload[:state]).to eq(:processing) 161 | when 'taskinator.task.processing' 162 | expect(payload[:state]).to eq(:processing) 163 | when 'taskinator.task.completed' 164 | expect(payload[:state]).to eq(:completed) 165 | when 'taskinator.process.completed' 166 | expect(payload[:state]).to eq(:completed) 167 | else 168 | raise "Unknown event '#{name}'." 169 | end 170 | 171 | end 172 | 173 | allow(Taskinator).to receive(:instrumenter).and_return(instrumenter) 174 | expect(instrumenter).to receive(:instrument).at_least(task_count).times.and_call_original 175 | 176 | expect(subject.current_state).to eq(:initial) 177 | 178 | subject.start! 179 | end 180 | 181 | end 182 | 183 | describe "job" do 184 | pending 185 | end 186 | 187 | describe "subprocess" do 188 | pending 189 | end 190 | end 191 | 192 | describe "instrumentation" do 193 | describe "task" do 194 | before do 195 | # override enqueue 196 | allow_any_instance_of(Taskinator::Task::Step).to receive(:enqueue!) { |task| 197 | # emulate the worker starting the task 198 | task.start! 199 | } 200 | end 201 | 202 | let(:task_count) { 10 } 203 | let(:definition) { TestFlows::Task } 204 | subject { definition.create_process(task_count) } 205 | 206 | it "reports task completed" do 207 | block = SpecSupport::Block.new 208 | expect(block).to receive(:call).exactly(task_count).times 209 | 210 | TestInstrumenter.subscribe(block, /taskinator.task.completed/) do 211 | subject.start! 212 | end 213 | end 214 | 215 | it "reports process completed" do 216 | block = SpecSupport::Block.new 217 | expect(block).to receive(:call).once 218 | 219 | TestInstrumenter.subscribe(block, /taskinator.process.completed/) do 220 | subject.start! 221 | end 222 | end 223 | 224 | it "reports task percentage completed" do 225 | invoke_count = 0 226 | 227 | instrumenter = TestInstrumenter.new do |name, payload| 228 | if name =~ /taskinator.task.processing/ 229 | expect(payload[:percentage_completed]).to eq( (invoke_count / task_count.to_f) * 100.0 ) 230 | elsif name =~ /taskinator.task.completed/ 231 | invoke_count += 1 232 | expect(payload[:percentage_completed]).to eq( (invoke_count / task_count.to_f) * 100.0 ) 233 | end 234 | end 235 | 236 | allow(Taskinator).to receive(:instrumenter).and_return(instrumenter) 237 | expect(instrumenter).to receive(:instrument).at_least(task_count).times.and_call_original 238 | 239 | subject.start! 240 | end 241 | 242 | it "reports process percentage completed" do 243 | instrumenter = TestInstrumenter.new do |name, payload| 244 | if name =~ /taskinator.process.started/ 245 | expect(payload[:process_uuid]).to eq(subject.uuid) 246 | elsif name =~ /taskinator.process.completed/ 247 | expect(payload[:percentage_completed]).to eq(100.0) 248 | end 249 | end 250 | 251 | allow(Taskinator).to receive(:instrumenter).and_return(instrumenter) 252 | expect(instrumenter).to receive(:instrument).at_least(task_count).times.and_call_original 253 | 254 | subject.start! 255 | end 256 | 257 | end 258 | 259 | describe "job" do 260 | before do 261 | # override enqueue 262 | allow_any_instance_of(Taskinator::Task::Job).to receive(:enqueue!) { |task| 263 | # emulate the worker starting the task 264 | task.start! 265 | } 266 | end 267 | 268 | let(:task_count) { 10 } 269 | let(:definition) { TestFlows::Job } 270 | subject { definition.create_process(task_count) } 271 | 272 | it "reports task completed" do 273 | block = SpecSupport::Block.new 274 | expect(block).to receive(:call).exactly(task_count).times 275 | 276 | TestInstrumenter.subscribe(block, /taskinator.task.completed/) do 277 | subject.start! 278 | end 279 | end 280 | 281 | it "reports process completed" do 282 | block = SpecSupport::Block.new 283 | expect(block).to receive(:call).once 284 | 285 | TestInstrumenter.subscribe(block, /taskinator.process.completed/) do 286 | subject.start! 287 | end 288 | end 289 | 290 | it "reports task percentage completed" do 291 | invoke_count = 0 292 | 293 | instrumenter = TestInstrumenter.new do |name, payload| 294 | if name =~ /taskinator.task.processing/ 295 | expect(payload[:percentage_completed]).to eq( (invoke_count / task_count.to_f) * 100.0 ) 296 | elsif name =~ /taskinator.task.completed/ 297 | invoke_count += 1 298 | expect(payload[:percentage_completed]).to eq( (invoke_count / task_count.to_f) * 100.0 ) 299 | end 300 | end 301 | 302 | allow(Taskinator).to receive(:instrumenter).and_return(instrumenter) 303 | expect(instrumenter).to receive(:instrument).at_least(task_count).times.and_call_original 304 | 305 | subject.start! 306 | end 307 | 308 | it "reports process percentage completed" do 309 | instrumenter = TestInstrumenter.new do |name, payload| 310 | if name =~ /taskinator.process.started/ 311 | expect(payload[:process_uuid]).to eq(subject.uuid) 312 | elsif name =~ /taskinator.process.completed/ 313 | expect(payload[:percentage_completed]).to eq(100.0) 314 | end 315 | end 316 | 317 | allow(Taskinator).to receive(:instrumenter).and_return(instrumenter) 318 | expect(instrumenter).to receive(:instrument).at_least(task_count).times.and_call_original 319 | 320 | subject.start! 321 | end 322 | 323 | end 324 | 325 | describe "sub process" do 326 | before do 327 | # override enqueue 328 | allow_any_instance_of(Taskinator::Task::Step).to receive(:enqueue!) { |task| 329 | # emulate the worker starting the task 330 | task.start! 331 | } 332 | 333 | # override enqueue 334 | allow_any_instance_of(Taskinator::Task::SubProcess).to receive(:enqueue!) { |task| 335 | # emulate the worker starting the task 336 | task.start! 337 | } 338 | end 339 | 340 | let(:task_count) { 10 } 341 | let(:definition) { TestFlows::SubProcess } 342 | subject { definition.create_process(task_count) } 343 | 344 | it "reports task completed" do 345 | block = SpecSupport::Block.new 346 | 347 | # NOTE: sub process counts for one task 348 | expect(block).to receive(:call).exactly(task_count + 1).times 349 | 350 | TestInstrumenter.subscribe(block, /taskinator.task.completed/) do 351 | subject.start! 352 | end 353 | end 354 | 355 | it "reports process completed" do 356 | block = SpecSupport::Block.new 357 | expect(block).to receive(:call).twice # includes sub process 358 | 359 | TestInstrumenter.subscribe(block, /taskinator.process.completed/) do 360 | subject.start! 361 | end 362 | end 363 | 364 | it "reports task percentage completed" do 365 | invoke_count = 0 366 | 367 | instrumenter = TestInstrumenter.new do |name, payload| 368 | if name =~ /taskinator.task.processing/ 369 | expect(payload[:percentage_completed]).to eq( (invoke_count / task_count.to_f) * 100.0 ) 370 | elsif name =~ /taskinator.task.completed/ 371 | unless payload.type.constantize >= Taskinator::Task::SubProcess 372 | invoke_count += 1 373 | expect(payload[:percentage_completed]).to eq( (invoke_count / task_count.to_f) * 100.0 ) 374 | end 375 | end 376 | end 377 | 378 | allow(Taskinator).to receive(:instrumenter).and_return(instrumenter) 379 | expect(instrumenter).to receive(:instrument).at_least(task_count).times.and_call_original 380 | 381 | subject.start! 382 | end 383 | 384 | it "reports process percentage completed" do 385 | instrumenter = TestInstrumenter.new do |name, payload| 386 | if name =~ /taskinator.process.started/ 387 | expect(payload[:process_uuid]).to eq(subject.uuid) 388 | elsif name =~ /taskinator.process.completed/ 389 | expect(payload[:percentage_completed]).to eq(100.0) 390 | end 391 | end 392 | 393 | allow(Taskinator).to receive(:instrumenter).and_return(instrumenter) 394 | 395 | # NOTE: sub process counts for one task 396 | expect(instrumenter).to receive(:instrument).at_least(task_count + 1).times.and_call_original 397 | 398 | subject.start! 399 | end 400 | 401 | end 402 | end 403 | 404 | end 405 | -------------------------------------------------------------------------------- /spec/taskinator/visitor_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Taskinator::Visitor::Base do 4 | 5 | it { respond_to(:visit_process) } 6 | it { respond_to(:visit_tasks) } 7 | it { respond_to(:visit_attribute) } 8 | it { respond_to(:visit_process_reference) } 9 | it { respond_to(:visit_task_reference) } 10 | it { respond_to(:visit_type) } 11 | it { respond_to(:visit_args) } 12 | it { respond_to(:task_count) } 13 | 14 | end 15 | -------------------------------------------------------------------------------- /taskinator.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'taskinator/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = 'taskinator' 8 | spec.version = Taskinator::VERSION 9 | spec.authors = ['Chris Stefano'] 10 | spec.email = ['virtualstaticvoid@gmail.com'] 11 | spec.description = %q{Simple process orchestration} 12 | spec.summary = %q{A simple orchestration library for running complex processes or workflows in Ruby} 13 | spec.homepage = 'https://github.com/virtualstaticvoid/taskinator' 14 | spec.license = 'MIT' 15 | 16 | spec.files = `git ls-files`.split($/) 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ['lib'] 20 | 21 | spec.required_ruby_version = '>= 2.0.0' 22 | 23 | # core 24 | spec.add_dependency 'redis' , '>= 3.2.1' 25 | spec.add_dependency 'redis-namespace' , '>= 1.5.2' 26 | spec.add_dependency 'connection_pool' , '>= 2.2.0' 27 | spec.add_dependency 'json' , '>= 1.8.2' 28 | spec.add_dependency 'builder' , '>= 3.2.2' 29 | spec.add_dependency 'globalid' , '>= 0.3' 30 | spec.add_dependency 'thwait' , '>= 0.2' 31 | 32 | end 33 | -------------------------------------------------------------------------------- /tasks_workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/virtualstaticvoid/taskinator/cc8408de02dc931c3d699888f20ac3be2aaacf27/tasks_workflow.png --------------------------------------------------------------------------------