├── .github └── workflows │ └── actions.yml ├── .gitignore ├── .rubocop.yml ├── Gemfile ├── Gemfile.lock ├── MIT-LICENSE.txt ├── Rakefile ├── Readme.md ├── lib ├── parallel.rb └── parallel │ └── version.rb ├── parallel.gemspec └── spec ├── cases ├── after_interrupt.rb ├── all_false.rb ├── all_true.rb ├── any_false.rb ├── any_true.rb ├── closes_processes_at_runtime.rb ├── count_open_pipes.rb ├── double_interrupt.rb ├── each.rb ├── each_in_place.rb ├── each_with_ar_sqlite.rb ├── each_with_index.rb ├── eof_in_process.rb ├── exception_raised_in_process.rb ├── exit_in_process.rb ├── fatal_queue.rb ├── filter_map.rb ├── finish_in_order.rb ├── flat_map.rb ├── helper.rb ├── map_isolation.rb ├── map_with_ar.rb ├── map_with_index.rb ├── map_with_index_empty.rb ├── map_with_killed_worker_before_read.rb ├── map_with_killed_worker_before_write.rb ├── map_with_nested_arrays_and_nil.rb ├── map_with_ractor.rb ├── map_worker_number_isolation.rb ├── no_dump_with_each.rb ├── no_gc_with_each.rb ├── parallel_break_better_errors.rb ├── parallel_fast_exit.rb ├── parallel_high_fork_rate.rb ├── parallel_influence_outside_data.rb ├── parallel_kill.rb ├── parallel_map.rb ├── parallel_map_complex_objects.rb ├── parallel_map_range.rb ├── parallel_map_sleeping.rb ├── parallel_map_uneven.rb ├── parallel_raise.rb ├── parallel_raise_undumpable.rb ├── parallel_sleeping_2.rb ├── parallel_start_and_kill.rb ├── parallel_with_detected_cpus.rb ├── parallel_with_nil_uses_detected_cpus.rb ├── parallel_with_set_processes.rb ├── profile_memory.rb ├── progress.rb ├── progress_with_finish.rb ├── progress_with_options.rb ├── synchronizes_start_and_finish.rb ├── timeout_in_threads.rb ├── with_break.rb ├── with_break_before_finish.rb ├── with_exception.rb ├── with_exception_before_finish.rb ├── with_exception_in_finish.rb ├── with_exception_in_start.rb ├── with_exception_in_start_before_finish.rb ├── with_lambda.rb ├── with_queue.rb └── with_worker_number.rb ├── parallel_spec.rb └── spec_helper.rb /.github/workflows/actions.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | branches: [master] 5 | pull_request: 6 | branches: [master] 7 | jobs: 8 | test: 9 | runs-on: ubuntu-latest 10 | services: 11 | mysql: 12 | image: mysql 13 | strategy: 14 | matrix: 15 | ruby: [ '2.7', '3.0', '3.1', '3.2', '3.3', '3.4' ] 16 | task: [ 'spec' ] 17 | include: 18 | - ruby: 2.7 # keep in sync with lowest version 19 | task: rubocop 20 | name: ${{ matrix.ruby }} rake ${{ matrix.task }} 21 | steps: 22 | - uses: actions/checkout@v4 23 | - uses: ruby/setup-ruby@v1 24 | with: 25 | ruby-version: ${{ matrix.ruby }} 26 | bundler-cache: true # runs 'bundle install' and caches installed gems automatically 27 | - run: bundle exec rake ${{ matrix.task }} 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /rspec.failures 2 | .ruby-version 3 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | plugins: 2 | - rubocop-rake 3 | - rubocop-rspec 4 | 5 | AllCops: 6 | NewCops: enable 7 | TargetRubyVersion: 2.7 8 | 9 | Style/StringLiterals: 10 | Enabled: false 11 | 12 | Bundler/OrderedGems: 13 | Enabled: false 14 | 15 | Metrics: 16 | Enabled: false 17 | 18 | Style/Documentation: 19 | Enabled: false 20 | 21 | Layout/EmptyLineAfterMagicComment: 22 | Enabled: false 23 | 24 | Layout/EndAlignment: 25 | EnforcedStyleAlignWith: variable 26 | 27 | Layout/MultilineOperationIndentation: 28 | Enabled: false 29 | 30 | Layout/MultilineMethodCallIndentation: 31 | EnforcedStyle: indented 32 | 33 | Style/NumericPredicate: 34 | EnforcedStyle: comparison 35 | 36 | Layout/EmptyLineAfterGuardClause: 37 | Enabled: false 38 | 39 | Layout/FirstHashElementLineBreak: 40 | Enabled: true # Opt-in 41 | 42 | # Opt-in 43 | Layout/FirstMethodArgumentLineBreak: 44 | Enabled: true # Opt-in 45 | 46 | Layout/FirstMethodParameterLineBreak: 47 | Enabled: true # Opt-in 48 | 49 | # https://github.com/rubocop-hq/rubocop/issues/5891 50 | Style/SpecialGlobalVars: 51 | Enabled: false 52 | 53 | Style/WordArray: 54 | EnforcedStyle: brackets 55 | 56 | Style/SymbolArray: 57 | EnforcedStyle: brackets 58 | 59 | Style/GuardClause: 60 | Enabled: false 61 | 62 | Style/EmptyElse: 63 | Enabled: false 64 | 65 | RSpec/DescribedClass: 66 | EnforcedStyle: explicit 67 | 68 | Style/DoubleNegation: 69 | Enabled: false 70 | 71 | RSpec/VerifiedDoubles: 72 | Enabled: false 73 | 74 | RSpec/ExampleLength: 75 | Enabled: false 76 | 77 | # does not understand .should 78 | RSpec/NoExpectationExample: 79 | Enabled: false 80 | 81 | Style/CombinableLoops: 82 | Enabled: false 83 | 84 | Lint/Void: 85 | Enabled: false 86 | 87 | Security/MarshalLoad: 88 | Enabled: false 89 | 90 | Lint/EmptyBlock: 91 | Exclude: [spec/**/*.rb] 92 | 93 | Naming/MethodParameterName: 94 | Exclude: [spec/**/*.rb] 95 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | source "https://rubygems.org" 3 | gemspec 4 | 5 | gem 'bump' 6 | gem 'rake' 7 | gem 'rspec' 8 | gem 'activerecord', "~> 6.0" 9 | gem 'ruby-progressbar' 10 | gem 'rspec-rerun' 11 | gem 'rspec-legacy_formatters' 12 | gem 'rubocop' 13 | gem 'rubocop-rake' 14 | gem 'rubocop-rspec' 15 | gem 'benchmark' 16 | gem 'logger' 17 | gem 'mutex_m' 18 | gem 'base64' 19 | gem 'bigdecimal' 20 | 21 | gem 'mysql2', group: :mysql 22 | gem 'sqlite3', '~> 1.4' 23 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | parallel (1.27.0) 5 | 6 | GEM 7 | remote: https://rubygems.org/ 8 | specs: 9 | activemodel (6.1.5) 10 | activesupport (= 6.1.5) 11 | activerecord (6.1.5) 12 | activemodel (= 6.1.5) 13 | activesupport (= 6.1.5) 14 | activesupport (6.1.5) 15 | concurrent-ruby (~> 1.0, >= 1.0.2) 16 | i18n (>= 1.6, < 2) 17 | minitest (>= 5.1) 18 | tzinfo (~> 2.0) 19 | zeitwerk (~> 2.3) 20 | ast (2.4.3) 21 | base64 (0.2.0) 22 | benchmark (0.4.0) 23 | bigdecimal (3.1.9) 24 | bigdecimal (3.1.9-java) 25 | bump (0.10.0) 26 | concurrent-ruby (1.1.9) 27 | diff-lcs (1.5.0) 28 | i18n (1.10.0) 29 | concurrent-ruby (~> 1.0) 30 | json (2.10.2) 31 | json (2.10.2-java) 32 | language_server-protocol (3.17.0.4) 33 | lint_roller (1.1.0) 34 | logger (1.7.0) 35 | mini_portile2 (2.8.7) 36 | minitest (5.15.0) 37 | mutex_m (0.3.0) 38 | mysql2 (0.5.6) 39 | parser (3.3.8.0) 40 | ast (~> 2.4.1) 41 | racc 42 | prism (1.4.0) 43 | racc (1.8.1) 44 | racc (1.8.1-java) 45 | rainbow (3.1.1) 46 | rake (13.0.6) 47 | regexp_parser (2.10.0) 48 | rspec (3.11.0) 49 | rspec-core (~> 3.11.0) 50 | rspec-expectations (~> 3.11.0) 51 | rspec-mocks (~> 3.11.0) 52 | rspec-core (3.11.0) 53 | rspec-support (~> 3.11.0) 54 | rspec-expectations (3.11.0) 55 | diff-lcs (>= 1.2.0, < 2.0) 56 | rspec-support (~> 3.11.0) 57 | rspec-legacy_formatters (1.0.2) 58 | rspec (~> 3.0) 59 | rspec-mocks (3.11.0) 60 | diff-lcs (>= 1.2.0, < 2.0) 61 | rspec-support (~> 3.11.0) 62 | rspec-rerun (1.1.0) 63 | rspec (~> 3.0) 64 | rspec-support (3.11.0) 65 | rubocop (1.75.2) 66 | json (~> 2.3) 67 | language_server-protocol (~> 3.17.0.2) 68 | lint_roller (~> 1.1.0) 69 | parallel (~> 1.10) 70 | parser (>= 3.3.0.2) 71 | rainbow (>= 2.2.2, < 4.0) 72 | regexp_parser (>= 2.9.3, < 3.0) 73 | rubocop-ast (>= 1.44.0, < 2.0) 74 | ruby-progressbar (~> 1.7) 75 | unicode-display_width (>= 2.4.0, < 4.0) 76 | rubocop-ast (1.44.1) 77 | parser (>= 3.3.7.2) 78 | prism (~> 1.4) 79 | rubocop-rake (0.7.1) 80 | lint_roller (~> 1.1) 81 | rubocop (>= 1.72.1) 82 | rubocop-rspec (3.5.0) 83 | lint_roller (~> 1.1) 84 | rubocop (~> 1.72, >= 1.72.1) 85 | ruby-progressbar (1.13.0) 86 | sqlite3 (1.7.3) 87 | mini_portile2 (~> 2.8.0) 88 | tzinfo (2.0.4) 89 | concurrent-ruby (~> 1.0) 90 | unicode-display_width (3.1.4) 91 | unicode-emoji (~> 4.0, >= 4.0.4) 92 | unicode-emoji (4.0.4) 93 | zeitwerk (2.5.4) 94 | 95 | PLATFORMS 96 | java 97 | ruby 98 | 99 | DEPENDENCIES 100 | activerecord (~> 6.0) 101 | base64 102 | benchmark 103 | bigdecimal 104 | bump 105 | logger 106 | mutex_m 107 | mysql2 108 | parallel! 109 | rake 110 | rspec 111 | rspec-legacy_formatters 112 | rspec-rerun 113 | rubocop 114 | rubocop-rake 115 | rubocop-rspec 116 | ruby-progressbar 117 | sqlite3 (~> 1.4) 118 | 119 | BUNDLED WITH 120 | 2.3.12 121 | -------------------------------------------------------------------------------- /MIT-LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (C) 2013 Michael Grosser 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'bundler/setup' 3 | require 'bundler/gem_tasks' 4 | require 'bump/tasks' 5 | require "rspec/core/rake_task" 6 | require 'rspec-rerun/tasks' 7 | 8 | task default: ["spec", "rubocop"] 9 | 10 | desc "Run tests" 11 | task spec: "rspec-rerun:spec" 12 | 13 | desc "Run rubocop" 14 | task :rubocop do 15 | sh "rubocop --parallel" 16 | end 17 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | Parallel 2 | ============== 3 | [![Gem Version](https://badge.fury.io/rb/parallel.svg)](https://rubygems.org/gems/parallel) 4 | [![Build Status](https://github.com/grosser/parallel/actions/workflows/actions.yml/badge.svg)](https://github.com/grosser/parallel/actions/workflows/actions.yml) 5 | 6 | 7 | Run any code in parallel Processes(> use all CPUs), Threads(> speedup blocking operations), or Ractors(> use all CPUs).
8 | Best suited for map-reduce or e.g. parallel downloads/uploads. 9 | 10 | Install 11 | ======= 12 | 13 | ```Bash 14 | gem install parallel 15 | ``` 16 | 17 | Usage 18 | ===== 19 | 20 | ```Ruby 21 | # 2 CPUs -> work in 2 processes (a,b + c) 22 | results = Parallel.map(['a','b','c']) do |one_letter| 23 | SomeClass.expensive_calculation(one_letter) 24 | end 25 | 26 | # 3 Processes -> finished after 1 run 27 | results = Parallel.map(['a','b','c'], in_processes: 3) { |one_letter| SomeClass.expensive_calculation(one_letter) } 28 | 29 | # 3 Threads -> finished after 1 run 30 | results = Parallel.map(['a','b','c'], in_threads: 3) { |one_letter| SomeClass.expensive_calculation(one_letter) } 31 | 32 | # 3 Ractors -> finished after 1 run 33 | results = Parallel.map(['a','b','c'], in_ractors: 3, ractor: [SomeClass, :expensive_calculation]) 34 | ``` 35 | 36 | Same can be done with `each` 37 | ```Ruby 38 | Parallel.each(['a','b','c']) { |one_letter| ... } 39 | ``` 40 | or `each_with_index`, `map_with_index`, `flat_map` 41 | 42 | Produce one item at a time with `lambda` (anything that responds to `.call`) or `Queue`. 43 | 44 | ```Ruby 45 | items = [1,2,3] 46 | Parallel.each( -> { items.pop || Parallel::Stop }) { |number| ... } 47 | ``` 48 | 49 | Also supports `any?` or `all?` 50 | 51 | ```Ruby 52 | Parallel.any?([1,2,3,4,5,6,7]) { |number| number == 4 } 53 | # => true 54 | 55 | Parallel.all?([1,2,nil,4,5]) { |number| number != nil } 56 | # => false 57 | ``` 58 | 59 | Processes/Threads are workers, they grab the next piece of work when they finish. 60 | 61 | ### Processes 62 | - Speedup through multiple CPUs 63 | - Speedup for blocking operations 64 | - Variables are protected from change 65 | - Extra memory used 66 | - Child processes are killed when your main process is killed through Ctrl+c or kill -2 67 | 68 | ### Threads 69 | - Speedup for blocking operations 70 | - Variables can be shared/modified 71 | - No extra memory used 72 | 73 | ### Ractors 74 | - Ruby 3.0+ only 75 | - Speedup for blocking operations 76 | - No extra memory used 77 | - Very fast to spawn 78 | - Experimental and unstable 79 | - `start` and `finish` hooks are called on main thread 80 | - Variables must be passed in `Parallel.map([1,2,3].map { |i| [i, ARGV, local_var] }, ...` 81 | - use `Ractor.make_shareable` to pass in global objects 82 | 83 | ### ActiveRecord 84 | 85 | #### Connection Lost 86 | 87 | - Multithreading needs connection pooling, forks need reconnects 88 | - Adjust connection pool size in `config/database.yml` when multithreading 89 | 90 | ```Ruby 91 | # reproducibly fixes things (spec/cases/map_with_ar.rb) 92 | Parallel.each(User.all, in_processes: 8) do |user| 93 | user.update_attribute(:some_attribute, some_value) 94 | end 95 | User.connection.reconnect! 96 | 97 | # maybe helps: explicitly use connection pool 98 | Parallel.each(User.all, in_threads: 8) do |user| 99 | ActiveRecord::Base.connection_pool.with_connection do 100 | user.update_attribute(:some_attribute, some_value) 101 | end 102 | end 103 | 104 | # maybe helps: reconnect once inside every fork 105 | Parallel.each(User.all, in_processes: 8) do |user| 106 | @reconnected ||= User.connection.reconnect! || true 107 | user.update_attribute(:some_attribute, some_value) 108 | end 109 | ``` 110 | 111 | #### NameError: uninitialized constant 112 | 113 | A race happens when ActiveRecord models are autoloaded inside parallel threads 114 | in environments that lazy-load, like development, test, or migrations. 115 | 116 | To fix, autoloaded classes before the parallel block with either `require ''` or `ModelName.class`. 117 | 118 | ### Break 119 | 120 | ```Ruby 121 | Parallel.map([1, 2, 3]) do |i| 122 | raise Parallel::Break # -> stops after all current items are finished 123 | end 124 | ``` 125 | 126 | ```Ruby 127 | Parallel.map([1, 2, 3]) { |i| raise Parallel::Break, i if i == 2 } == 2 128 | ``` 129 | 130 | ### Kill 131 | 132 | Only use if whatever is executing in the sub-command is safe to kill at any point 133 | 134 | ```Ruby 135 | Parallel.map([1,2,3]) do |x| 136 | raise Parallel::Kill if x == 1# -> stop all sub-processes, killing them instantly 137 | sleep 100 # Do stuff 138 | end 139 | ``` 140 | 141 | ### Progress / ETA 142 | 143 | ```Ruby 144 | # gem install ruby-progressbar 145 | 146 | Parallel.map(1..50, progress: "Doing stuff") { sleep 1 } 147 | 148 | # Doing stuff | ETA: 00:00:02 | ==================== | Time: 00:00:10 149 | ``` 150 | 151 | Use `:finish` or `:start` hook to get progress information. 152 | - `:start` has item and index 153 | - `:finish` has item, index, and result 154 | 155 | They are called on the main process and protected with a mutex. 156 | (To just get the index, use the more performant `Parallel.each_with_index`) 157 | 158 | ```Ruby 159 | Parallel.map(1..100, finish: -> (item, i, result) { ... do something ... }) { sleep 1 } 160 | ``` 161 | 162 | Set `finish_in_order: true` to call the `:finish` hook in the order of the input (will take longer to see initial output). 163 | 164 | ```Ruby 165 | Parallel.map(1..9, finish: -> (item, i, result) { puts "#{item} ok" }, finish_in_order: true) { sleep rand } 166 | ``` 167 | 168 | ### Worker number 169 | 170 | Use `Parallel.worker_number` to determine the worker slot in which your 171 | task is running. 172 | 173 | ```Ruby 174 | Parallel.each(1..5, in_processes: 2) { |i| puts "Item: #{i}, Worker: #{Parallel.worker_number}" } 175 | Item: 1, Worker: 1 176 | Item: 2, Worker: 0 177 | Item: 3, Worker: 1 178 | Item: 4, Worker: 0 179 | Item: 5, Worker: 1 180 | ``` 181 | 182 | ### Dynamically generating jobs 183 | 184 | Example: wait for work to arrive or sleep 185 | 186 | ```ruby 187 | queue = [] 188 | Thread.new { loop { queue << rand(100); sleep 2 } } # job producer 189 | Parallel.map(Proc.new { queue.pop }, in_processes: 3) { |f| f ? puts("#{f} received") : sleep(1) } 190 | ``` 191 | 192 | Tips 193 | ==== 194 | 195 | - [Benchmark/Test] Disable threading/forking with `in_threads: 0` or `in_processes: 0`, to run the same code with different setups 196 | - [Isolation] Do not reuse previous worker processes: `isolation: true` 197 | - [Stop all processes with an alternate interrupt signal] `'INT'` (from `ctrl+c`) is caught by default. Catch `'TERM'` (from `kill`) with `interrupt_signal: 'TERM'` 198 | - [Process count via ENV] `PARALLEL_PROCESSOR_COUNT=16` will use `16` instead of the number of processors detected. This is used to reconfigure a tool using `parallel` without inserting custom logic. 199 | - [Process count] `parallel` uses a number of processors seen by the OS for process count by default. If you want to use a value considering CPU quota, please add `concurrent-ruby` to your `Gemfile`. 200 | 201 | TODO 202 | ==== 203 | - Replace Signal trapping with simple `rescue Interrupt` handler 204 | 205 | Authors 206 | ======= 207 | 208 | ### [Contributors](https://github.com/grosser/parallel/graphs/contributors) 209 | - [Przemyslaw Wroblewski](https://github.com/lowang) 210 | - [TJ Holowaychuk](http://vision-media.ca/) 211 | - [Masatomo Nakano](https://github.com/masatomo) 212 | - [Fred Wu](http://fredwu.me) 213 | - [mikezter](https://github.com/mikezter) 214 | - [Jeremy Durham](http://www.jeremydurham.com) 215 | - [Nick Gauthier](http://www.ngauthier.com) 216 | - [Andrew Bowerman](http://andrewbowerman.com) 217 | - [Byron Bowerman](http://blog.bm5k.com/) 218 | - [Mikko Kokkonen](https://github.com/mikian) 219 | - [brian p o'rourke](https://github.com/bpo) 220 | - [Norio Sato] 221 | - [Neal Stewart](https://github.com/n-time) 222 | - [Jurriaan Pruis](https://github.com/jurriaan) 223 | - [Rob Worley](https://github.com/robworley) 224 | - [Tasveer Singh](https://github.com/tazsingh) 225 | - [Joachim](https://github.com/jmozmoz) 226 | - [yaoguai](https://github.com/yaoguai) 227 | - [Bartosz Dziewoński](https://github.com/MatmaRex) 228 | - [yaoguai](https://github.com/yaoguai) 229 | - [Guillaume Hain](https://github.com/zedtux) 230 | - [Adam Wróbel](https://github.com/amw) 231 | - [Matthew Brennan](https://github.com/mattyb) 232 | - [Brendan Dougherty](https://github.com/brendar) 233 | - [Daniel Finnie](https://github.com/danfinnie) 234 | - [Philip M. White](https://github.com/philipmw) 235 | - [Arlan Jaska](https://github.com/ajaska) 236 | - [Sean Walbran](https://github.com/seanwalbran) 237 | - [Nathan Broadbent](https://github.com/ndbroadbent) 238 | - [Yuki Inoue](https://github.com/Yuki-Inoue) 239 | - [Takumasa Ochi](https://github.com/aeroastro) 240 | - [Shai Coleman](https://github.com/shaicoleman) 241 | - [Earlopain](https://github.com/Earlopain) 242 | 243 | [Michael Grosser](http://grosser.it)
244 | michael@grosser.it
245 | License: MIT
246 | -------------------------------------------------------------------------------- /lib/parallel.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'rbconfig' 3 | require 'parallel/version' 4 | 5 | module Parallel 6 | Stop = Object.new.freeze 7 | 8 | class DeadWorker < StandardError 9 | end 10 | 11 | class Break < StandardError 12 | attr_reader :value 13 | 14 | def initialize(value = nil) 15 | super() 16 | @value = value 17 | end 18 | end 19 | 20 | class Kill < Break 21 | end 22 | 23 | class UndumpableException < StandardError 24 | attr_reader :backtrace 25 | 26 | def initialize(original) 27 | super("#{original.class}: #{original.message}") 28 | @backtrace = original.backtrace 29 | end 30 | end 31 | 32 | class ExceptionWrapper 33 | attr_reader :exception 34 | 35 | def initialize(exception) 36 | # Remove the bindings stack added by the better_errors gem, 37 | # because it cannot be marshalled 38 | if exception.instance_variable_defined? :@__better_errors_bindings_stack 39 | exception.send :remove_instance_variable, :@__better_errors_bindings_stack 40 | end 41 | 42 | @exception = 43 | begin 44 | Marshal.dump(exception) && exception 45 | rescue StandardError 46 | UndumpableException.new(exception) 47 | end 48 | end 49 | end 50 | 51 | class Worker 52 | attr_reader :pid, :read, :write 53 | attr_accessor :thread 54 | 55 | def initialize(read, write, pid) 56 | @read = read 57 | @write = write 58 | @pid = pid 59 | end 60 | 61 | def stop 62 | close_pipes 63 | wait # if it goes zombie, rather wait here to be able to debug 64 | end 65 | 66 | # might be passed to started_processes and simultaneously closed by another thread 67 | # when running in isolation mode, so we have to check if it is closed before closing 68 | def close_pipes 69 | read.close unless read.closed? 70 | write.close unless write.closed? 71 | end 72 | 73 | def work(data) 74 | begin 75 | Marshal.dump(data, write) 76 | rescue Errno::EPIPE 77 | raise DeadWorker 78 | end 79 | 80 | result = begin 81 | Marshal.load(read) 82 | rescue EOFError 83 | raise DeadWorker 84 | end 85 | raise result.exception if result.is_a?(ExceptionWrapper) 86 | result 87 | end 88 | 89 | private 90 | 91 | def wait 92 | Process.wait(pid) 93 | rescue Interrupt 94 | # process died 95 | end 96 | end 97 | 98 | class JobFactory 99 | def initialize(source, mutex) 100 | @lambda = (source.respond_to?(:call) && source) || queue_wrapper(source) 101 | @source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array 102 | @mutex = mutex 103 | @index = -1 104 | @stopped = false 105 | end 106 | 107 | def next 108 | if producer? 109 | # - index and item stay in sync 110 | # - do not call lambda after it has returned Stop 111 | item, index = @mutex.synchronize do 112 | return if @stopped 113 | item = @lambda.call 114 | @stopped = (item == Stop) 115 | return if @stopped 116 | [item, @index += 1] 117 | end 118 | else 119 | index = @mutex.synchronize { @index += 1 } 120 | return if index >= size 121 | item = @source[index] 122 | end 123 | [item, index] 124 | end 125 | 126 | def size 127 | if producer? 128 | Float::INFINITY 129 | else 130 | @source.size 131 | end 132 | end 133 | 134 | # generate item that is sent to workers 135 | # just index is faster + less likely to blow up with unserializable errors 136 | def pack(item, index) 137 | producer? ? [item, index] : index 138 | end 139 | 140 | # unpack item that is sent to workers 141 | def unpack(data) 142 | producer? ? data : [@source[data], data] 143 | end 144 | 145 | private 146 | 147 | def producer? 148 | @lambda 149 | end 150 | 151 | def queue_wrapper(array) 152 | array.respond_to?(:num_waiting) && array.respond_to?(:pop) && -> { array.pop(false) } 153 | end 154 | end 155 | 156 | class UserInterruptHandler 157 | INTERRUPT_SIGNAL = :SIGINT 158 | 159 | class << self 160 | # kill all these pids or threads if user presses Ctrl+c 161 | def kill_on_ctrl_c(pids, options) 162 | @to_be_killed ||= [] 163 | old_interrupt = nil 164 | signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL) 165 | 166 | if @to_be_killed.empty? 167 | old_interrupt = trap_interrupt(signal) do 168 | warn 'Parallel execution interrupted, exiting ...' 169 | @to_be_killed.flatten.each { |pid| kill(pid) } 170 | end 171 | end 172 | 173 | @to_be_killed << pids 174 | 175 | yield 176 | ensure 177 | @to_be_killed.pop # do not kill pids that could be used for new processes 178 | restore_interrupt(old_interrupt, signal) if @to_be_killed.empty? 179 | end 180 | 181 | def kill(thing) 182 | Process.kill(:KILL, thing) 183 | rescue Errno::ESRCH 184 | # some linux systems already automatically killed the children at this point 185 | # so we just ignore them not being there 186 | end 187 | 188 | private 189 | 190 | def trap_interrupt(signal) 191 | old = Signal.trap signal, 'IGNORE' 192 | 193 | Signal.trap signal do 194 | yield 195 | if !old || old == "DEFAULT" 196 | raise Interrupt 197 | else 198 | old.call 199 | end 200 | end 201 | 202 | old 203 | end 204 | 205 | def restore_interrupt(old, signal) 206 | Signal.trap signal, old 207 | end 208 | end 209 | end 210 | 211 | class << self 212 | def in_threads(options = { count: 2 }) 213 | threads = [] 214 | count, = extract_count_from_options(options) 215 | 216 | Thread.handle_interrupt(Exception => :never) do 217 | Thread.handle_interrupt(Exception => :immediate) do 218 | count.times do |i| 219 | threads << Thread.new { yield(i) } 220 | end 221 | threads.map(&:value) 222 | end 223 | ensure 224 | threads.each(&:kill) 225 | end 226 | end 227 | 228 | def in_processes(options = {}, &block) 229 | count, options = extract_count_from_options(options) 230 | count ||= processor_count 231 | map(0...count, options.merge(in_processes: count), &block) 232 | end 233 | 234 | def each(array, options = {}, &block) 235 | map(array, options.merge(preserve_results: false), &block) 236 | end 237 | 238 | def any?(*args, &block) 239 | raise "You must provide a block when calling #any?" if block.nil? 240 | !each(*args) { |*a| raise Kill if block.call(*a) } 241 | end 242 | 243 | def all?(*args, &block) 244 | raise "You must provide a block when calling #all?" if block.nil? 245 | !!each(*args) { |*a| raise Kill unless block.call(*a) } 246 | end 247 | 248 | def each_with_index(array, options = {}, &block) 249 | each(array, options.merge(with_index: true), &block) 250 | end 251 | 252 | def map(source, options = {}, &block) 253 | options = options.dup 254 | options[:mutex] = Mutex.new 255 | 256 | if options[:in_processes] && options[:in_threads] 257 | raise ArgumentError, "Please specify only one of `in_processes` or `in_threads`." 258 | elsif RUBY_PLATFORM =~ (/java/) && !options[:in_processes] 259 | method = :in_threads 260 | size = options[method] || processor_count 261 | elsif options[:in_threads] 262 | method = :in_threads 263 | size = options[method] 264 | elsif options[:in_ractors] 265 | method = :in_ractors 266 | size = options[method] 267 | else 268 | method = :in_processes 269 | if Process.respond_to?(:fork) 270 | size = options[method] || processor_count 271 | else 272 | warn "Process.fork is not supported by this Ruby" 273 | size = 0 274 | end 275 | end 276 | 277 | job_factory = JobFactory.new(source, options[:mutex]) 278 | size = [job_factory.size, size].min 279 | 280 | options[:return_results] = (options[:preserve_results] != false || !!options[:finish]) 281 | add_progress_bar!(job_factory, options) 282 | 283 | result = 284 | if size == 0 285 | work_direct(job_factory, options, &block) 286 | elsif method == :in_threads 287 | work_in_threads(job_factory, options.merge(count: size), &block) 288 | elsif method == :in_ractors 289 | work_in_ractors(job_factory, options.merge(count: size), &block) 290 | else 291 | work_in_processes(job_factory, options.merge(count: size), &block) 292 | end 293 | 294 | return result.value if result.is_a?(Break) 295 | raise result if result.is_a?(Exception) 296 | options[:return_results] ? result : source 297 | end 298 | 299 | def map_with_index(array, options = {}, &block) 300 | map(array, options.merge(with_index: true), &block) 301 | end 302 | 303 | def flat_map(...) 304 | map(...).flatten(1) 305 | end 306 | 307 | def filter_map(...) 308 | map(...).compact 309 | end 310 | 311 | # Number of physical processor cores on the current system. 312 | def physical_processor_count 313 | @physical_processor_count ||= begin 314 | ppc = 315 | case RbConfig::CONFIG["target_os"] 316 | when /darwin[12]/ 317 | IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i 318 | when /linux/ 319 | cores = {} # unique physical ID / core ID combinations 320 | phy = 0 321 | File.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln| 322 | if ln.start_with?("physical") 323 | phy = ln[/\d+/] 324 | elsif ln.start_with?("core") 325 | cid = "#{phy}:#{ln[/\d+/]}" 326 | cores[cid] = true unless cores[cid] 327 | end 328 | end 329 | cores.count 330 | when /mswin|mingw/ 331 | physical_processor_count_windows 332 | else 333 | processor_count 334 | end 335 | # fall back to logical count if physical info is invalid 336 | ppc > 0 ? ppc : processor_count 337 | end 338 | end 339 | 340 | # Number of processors seen by the OS or value considering CPU quota if the process is inside a cgroup, 341 | # used for process scheduling 342 | def processor_count 343 | @processor_count ||= Integer(ENV['PARALLEL_PROCESSOR_COUNT'] || available_processor_count) 344 | end 345 | 346 | def worker_number 347 | Thread.current[:parallel_worker_number] 348 | end 349 | 350 | # TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed 351 | def worker_number=(worker_num) 352 | Thread.current[:parallel_worker_number] = worker_num 353 | end 354 | 355 | private 356 | 357 | def physical_processor_count_windows 358 | # Get-CimInstance introduced in PowerShell 3 or earlier: https://learn.microsoft.com/en-us/previous-versions/powershell/module/cimcmdlets/get-ciminstance?view=powershell-3.0 359 | result = run( 360 | 'powershell -command "Get-CimInstance -ClassName Win32_Processor -Property NumberOfCores ' \ 361 | '| Select-Object -Property NumberOfCores"' 362 | ) 363 | if !result || $?.exitstatus != 0 364 | # fallback to deprecated wmic for older systems 365 | result = run("wmic cpu get NumberOfCores") 366 | end 367 | if !result || $?.exitstatus != 0 368 | # Bail out if both commands returned something unexpected 369 | warn "guessing pyhsical processor count" 370 | processor_count 371 | else 372 | # powershell: "\nNumberOfCores\n-------------\n 4\n\n\n" 373 | # wmic: "NumberOfCores \n\n4 \n\n\n\n" 374 | result.scan(/\d+/).map(&:to_i).reduce(:+) 375 | end 376 | end 377 | 378 | def run(command) 379 | IO.popen(command, &:read) 380 | rescue Errno::ENOENT 381 | # Ignore 382 | end 383 | 384 | def add_progress_bar!(job_factory, options) 385 | if (progress_options = options[:progress]) 386 | raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY 387 | require 'ruby-progressbar' 388 | 389 | if progress_options == true 390 | progress_options = { title: "Progress" } 391 | elsif progress_options.respond_to? :to_str 392 | progress_options = { title: progress_options.to_str } 393 | end 394 | 395 | progress_options = { 396 | total: job_factory.size, 397 | format: '%t |%E | %B | %a' 398 | }.merge(progress_options) 399 | 400 | progress = ProgressBar.create(progress_options) 401 | old_finish = options[:finish] 402 | options[:finish] = lambda do |item, i, result| 403 | old_finish.call(item, i, result) if old_finish 404 | progress.increment 405 | end 406 | end 407 | end 408 | 409 | def work_direct(job_factory, options, &block) 410 | self.worker_number = 0 411 | results = [] 412 | exception = nil 413 | begin 414 | while (set = job_factory.next) 415 | item, index = set 416 | results << with_instrumentation(item, index, options) do 417 | call_with_index(item, index, options, &block) 418 | end 419 | end 420 | rescue StandardError 421 | exception = $! 422 | end 423 | exception || results 424 | ensure 425 | self.worker_number = nil 426 | end 427 | 428 | def work_in_threads(job_factory, options, &block) 429 | raise "interrupt_signal is no longer supported for threads" if options[:interrupt_signal] 430 | results = [] 431 | results_mutex = Mutex.new # arrays are not thread-safe on jRuby 432 | exception = nil 433 | 434 | in_threads(options) do |worker_num| 435 | self.worker_number = worker_num 436 | # as long as there are more jobs, work on one of them 437 | while !exception && (set = job_factory.next) 438 | begin 439 | item, index = set 440 | result = with_instrumentation item, index, options do 441 | call_with_index(item, index, options, &block) 442 | end 443 | results_mutex.synchronize { results[index] = result } 444 | rescue StandardError 445 | exception = $! 446 | end 447 | end 448 | end 449 | 450 | exception || results 451 | end 452 | 453 | def work_in_ractors(job_factory, options) 454 | exception = nil 455 | results = [] 456 | results_mutex = Mutex.new # arrays are not thread-safe on jRuby 457 | 458 | callback = options[:ractor] 459 | if block_given? || !callback 460 | raise ArgumentError, "pass the code you want to execute as `ractor: [ClassName, :method_name]`" 461 | end 462 | 463 | # build 464 | ractors = Array.new(options.fetch(:count)) do 465 | Ractor.new do 466 | loop do 467 | got = receive 468 | (klass, method_name), item, index = got 469 | break if index == :break 470 | begin 471 | Ractor.yield [nil, klass.send(method_name, item), item, index] 472 | rescue StandardError => e 473 | Ractor.yield [e, nil, item, index] 474 | end 475 | end 476 | end 477 | end 478 | 479 | # start 480 | ractors.dup.each do |ractor| 481 | if (set = job_factory.next) 482 | item, index = set 483 | instrument_start item, index, options 484 | ractor.send [callback, item, index] 485 | else 486 | ractor.send([[nil, nil], nil, :break]) # stop the ractor 487 | ractors.delete ractor 488 | end 489 | end 490 | 491 | # replace with new items 492 | while (set = job_factory.next) 493 | item_next, index_next = set 494 | done, (exception, result, item, index) = Ractor.select(*ractors) 495 | if exception 496 | ractors.delete done 497 | break 498 | end 499 | instrument_finish item, index, result, options 500 | results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) } 501 | 502 | instrument_start item_next, index_next, options 503 | done.send([callback, item_next, index_next]) 504 | end 505 | 506 | # finish 507 | ractors.each do |ractor| 508 | (new_exception, result, item, index) = ractor.take 509 | exception ||= new_exception 510 | next if new_exception 511 | instrument_finish item, index, result, options 512 | results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) } 513 | ractor.send([[nil, nil], nil, :break]) # stop the ractor 514 | end 515 | 516 | exception || results 517 | end 518 | 519 | def work_in_processes(job_factory, options, &blk) 520 | workers = create_workers(job_factory, options, &blk) 521 | results = [] 522 | results_mutex = Mutex.new # arrays are not thread-safe 523 | exception = nil 524 | 525 | UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do 526 | in_threads(options) do |i| 527 | worker = workers[i] 528 | worker.thread = Thread.current 529 | worked = false 530 | 531 | begin 532 | loop do 533 | break if exception 534 | item, index = job_factory.next 535 | break unless index 536 | 537 | if options[:isolation] 538 | worker = replace_worker(job_factory, workers, i, options, blk) if worked 539 | worked = true 540 | worker.thread = Thread.current 541 | end 542 | 543 | begin 544 | result = with_instrumentation item, index, options do 545 | worker.work(job_factory.pack(item, index)) 546 | end 547 | results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby 548 | rescue StandardError => e 549 | exception = e 550 | if exception.is_a?(Kill) 551 | (workers - [worker]).each do |w| 552 | w.thread&.kill 553 | UserInterruptHandler.kill(w.pid) 554 | end 555 | end 556 | end 557 | end 558 | ensure 559 | worker.stop 560 | end 561 | end 562 | end 563 | 564 | exception || results 565 | end 566 | 567 | def replace_worker(job_factory, workers, index, options, blk) 568 | options[:mutex].synchronize do 569 | # old worker is no longer used ... stop it 570 | worker = workers[index] 571 | worker.stop 572 | 573 | # create a new replacement worker 574 | running = workers - [worker] 575 | workers[index] = worker(job_factory, options.merge(started_workers: running, worker_number: index), &blk) 576 | end 577 | end 578 | 579 | def create_workers(job_factory, options, &block) 580 | workers = [] 581 | Array.new(options[:count]).each_with_index do |_, i| 582 | workers << worker(job_factory, options.merge(started_workers: workers, worker_number: i), &block) 583 | end 584 | workers 585 | end 586 | 587 | def worker(job_factory, options, &block) 588 | child_read, parent_write = IO.pipe 589 | parent_read, child_write = IO.pipe 590 | 591 | pid = Process.fork do 592 | self.worker_number = options[:worker_number] 593 | 594 | begin 595 | options.delete(:started_workers).each(&:close_pipes) 596 | 597 | parent_write.close 598 | parent_read.close 599 | 600 | process_incoming_jobs(child_read, child_write, job_factory, options, &block) 601 | ensure 602 | child_read.close 603 | child_write.close 604 | end 605 | end 606 | 607 | child_read.close 608 | child_write.close 609 | 610 | Worker.new(parent_read, parent_write, pid) 611 | end 612 | 613 | def process_incoming_jobs(read, write, job_factory, options, &block) 614 | until read.eof? 615 | data = Marshal.load(read) 616 | item, index = job_factory.unpack(data) 617 | 618 | result = 619 | begin 620 | call_with_index(item, index, options, &block) 621 | # https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140 622 | rescue NoMemoryError, SignalException, Interrupt, SystemExit # rubocop:disable Lint/ShadowedException 623 | raise $! 624 | rescue Exception # # rubocop:disable Lint/RescueException 625 | ExceptionWrapper.new($!) 626 | end 627 | 628 | begin 629 | Marshal.dump(result, write) 630 | rescue Errno::EPIPE 631 | return # parent thread already dead 632 | end 633 | end 634 | end 635 | 636 | # options is either a Integer or a Hash with :count 637 | def extract_count_from_options(options) 638 | if options.is_a?(Hash) 639 | count = options[:count] 640 | else 641 | count = options 642 | options = {} 643 | end 644 | [count, options] 645 | end 646 | 647 | def call_with_index(item, index, options, &block) 648 | args = [item] 649 | args << index if options[:with_index] 650 | results = block.call(*args) 651 | if options[:return_results] 652 | results 653 | else 654 | nil # avoid GC overhead of passing large results around 655 | end 656 | end 657 | 658 | def with_instrumentation(item, index, options) 659 | instrument_start(item, index, options) 660 | result = yield 661 | instrument_finish(item, index, result, options) 662 | result unless options[:preserve_results] == false 663 | end 664 | 665 | def instrument_finish(item, index, result, options) 666 | return unless (on_finish = options[:finish]) 667 | return instrument_finish_in_order(item, index, result, options) if options[:finish_in_order] 668 | options[:mutex].synchronize { on_finish.call(item, index, result) } 669 | end 670 | 671 | # yield results in the order of the input items 672 | # needs to use `options` to store state between executions 673 | # needs to use `done` index since a nil result would also be valid 674 | def instrument_finish_in_order(item, index, result, options) 675 | options[:mutex].synchronize do 676 | # initialize our state 677 | options[:finish_done] ||= [] 678 | options[:finish_expecting] ||= 0 # we wait for item at index 0 679 | 680 | # store current result 681 | options[:finish_done][index] = [item, result] 682 | 683 | # yield all results that are now in order 684 | break unless index == options[:finish_expecting] 685 | index.upto(options[:finish_done].size).each do |i| 686 | break unless (done = options[:finish_done][i]) 687 | options[:finish_done][i] = nil # allow GC to free this item and result 688 | options[:finish].call(done[0], i, done[1]) 689 | options[:finish_expecting] += 1 690 | end 691 | end 692 | end 693 | 694 | def instrument_start(item, index, options) 695 | return unless (on_start = options[:start]) 696 | options[:mutex].synchronize { on_start.call(item, index) } 697 | end 698 | 699 | def available_processor_count 700 | gem 'concurrent-ruby', '>= 1.3.4' 701 | require 'concurrent-ruby' 702 | Concurrent.available_processor_count.floor 703 | rescue LoadError 704 | require 'etc' 705 | Etc.nprocessors 706 | end 707 | end 708 | end 709 | -------------------------------------------------------------------------------- /lib/parallel/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | module Parallel 3 | VERSION = Version = '1.27.0' # rubocop:disable Naming/ConstantName 4 | end 5 | -------------------------------------------------------------------------------- /parallel.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | name = "parallel" 3 | $LOAD_PATH << File.expand_path('lib', __dir__) 4 | require "#{name}/version" 5 | 6 | Gem::Specification.new name, Parallel::VERSION do |s| 7 | s.summary = "Run any kind of code in parallel processes" 8 | s.authors = ["Michael Grosser"] 9 | s.email = "michael@grosser.it" 10 | s.homepage = "https://github.com/grosser/#{name}" 11 | s.metadata = { 12 | "bug_tracker_uri" => "https://github.com/grosser/#{name}/issues", 13 | "documentation_uri" => "https://github.com/grosser/#{name}/blob/v#{s.version}/Readme.md", 14 | "source_code_uri" => "https://github.com/grosser/#{name}/tree/v#{s.version}", 15 | "wiki_uri" => "https://github.com/grosser/#{name}/wiki" 16 | } 17 | s.files = `git ls-files lib MIT-LICENSE.txt`.split("\n") 18 | s.license = "MIT" 19 | s.required_ruby_version = '>= 2.7' 20 | end 21 | -------------------------------------------------------------------------------- /spec/cases/after_interrupt.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | Parallel.map([1, 2], in_processes: 2) {} 5 | 6 | puts Signal.trap(:SIGINT, "IGNORE") 7 | -------------------------------------------------------------------------------- /spec/cases/all_false.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | $stdout.sync = true # otherwise results can go weird... 4 | 5 | results = [] 6 | 7 | [{ in_processes: 2 }, { in_threads: 2 }, { in_threads: 0 }].each do |options| 8 | x = [nil, nil, nil, nil, nil, nil, nil, nil] 9 | results << Parallel.all?(x, options) do |y| 10 | y 11 | end 12 | 13 | x = [42, 42, 42, 42, 42, 42, 42, 5, 42, 42, 42] 14 | results << Parallel.all?(x, options) do |y| 15 | y == 42 16 | end 17 | end 18 | 19 | print results.join(',') 20 | -------------------------------------------------------------------------------- /spec/cases/all_true.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | $stdout.sync = true # otherwise results can go weird... 4 | 5 | results = [] 6 | 7 | [{ in_processes: 2 }, { in_threads: 2 }, { in_threads: 0 }].each do |options| 8 | x = [nil, nil, nil, nil, nil, nil, nil, nil] 9 | results << Parallel.all?(x, options, &:nil?) 10 | 11 | x = [42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42] 12 | results << Parallel.all?(x, options) do |y| 13 | y == 42 14 | end 15 | 16 | # Empty array should return true 17 | x = [] 18 | results << Parallel.all?(x, options) do |y| 19 | y 20 | end 21 | end 22 | 23 | print results.join(',') 24 | -------------------------------------------------------------------------------- /spec/cases/any_false.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | $stdout.sync = true # otherwise results can go weird... 4 | 5 | results = [] 6 | 7 | [{ in_processes: 2 }, { in_threads: 2 }, { in_threads: 0 }].each do |options| 8 | x = [nil, nil, nil, nil, nil, nil, nil, nil] 9 | results << Parallel.any?(x, options) do |y| 10 | y 11 | end 12 | 13 | x = 10.times 14 | results << Parallel.any?(x, options) do |_y| 15 | false 16 | end 17 | 18 | # Empty array should return false 19 | x = [] 20 | results << Parallel.any?(x, options) do |y| 21 | y == 42 22 | end 23 | end 24 | 25 | print results.join(',') 26 | -------------------------------------------------------------------------------- /spec/cases/any_true.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | $stdout.sync = true # otherwise results can go weird... 4 | 5 | results = [] 6 | 7 | [{ in_processes: 2 }, { in_threads: 2 }, { in_threads: 0 }].each do |options| 8 | x = [nil, nil, nil, nil, 42, nil, nil, nil] 9 | results << Parallel.any?(x, options) do |y| 10 | y 11 | end 12 | 13 | x = [true, true, true, false, true, true, true] 14 | results << Parallel.any?(x, options) do |y| 15 | y 16 | end 17 | end 18 | 19 | print results.join(',') 20 | -------------------------------------------------------------------------------- /spec/cases/closes_processes_at_runtime.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | process_diff do 5 | Parallel.each((0..10).to_a, in_processes: 5) { |a| raise unless a * 2 } 6 | end 7 | -------------------------------------------------------------------------------- /spec/cases/count_open_pipes.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | count = ->(*) { `lsof -l | grep pipe | wc -l`.to_i } 4 | start = count.call 5 | results = Parallel.map(Array.new(20), in_processes: 20, &count) 6 | puts results.max - start 7 | -------------------------------------------------------------------------------- /spec/cases/double_interrupt.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | Signal.trap :SIGINT do 5 | sleep 0.5 6 | puts "YES" 7 | exit 0 8 | end 9 | 10 | Parallel.map(Array.new(20), in_processes: 2) do 11 | sleep 10 12 | puts "I should be killed earlier" 13 | end 14 | -------------------------------------------------------------------------------- /spec/cases/each.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | $stdout.sync = true # otherwise results can go weird... 4 | 5 | x = ['a', 'b', 'c', 'd'] 6 | result = Parallel.each(x) do |y| 7 | sleep 0.1 if y == 'a' 8 | end 9 | print result * ' ' 10 | -------------------------------------------------------------------------------- /spec/cases/each_in_place.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | $stdout.sync = true # otherwise results can go weird... 4 | 5 | x = [+'a'] 6 | Parallel.each(x, in_threads: 1) { |y| y << 'b' } 7 | print x.first 8 | -------------------------------------------------------------------------------- /spec/cases/each_with_ar_sqlite.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | require "logger" 4 | require "active_record" 5 | require "sqlite3" 6 | require "tempfile" 7 | $stdout.sync = true 8 | in_worker_type = :"in_#{ENV.fetch('WORKER_TYPE')}" 9 | 10 | Tempfile.create("db") do |temp| 11 | ActiveRecord::Schema.verbose = false 12 | ActiveRecord::Base.establish_connection( 13 | adapter: "sqlite3", 14 | database: temp.path 15 | ) 16 | 17 | class User < ActiveRecord::Base # rubocop:disable Lint/ConstantDefinitionInBlock 18 | end 19 | 20 | class Callback # rubocop:disable Lint/ConstantDefinitionInBlock 21 | def self.call(_) 22 | $stdout.sync = true 23 | puts "Parallel: #{User.all.map(&:name).join}" 24 | end 25 | end 26 | 27 | # create tables 28 | unless User.table_exists? 29 | ActiveRecord::Schema.define(version: 1) do 30 | create_table :users do |t| 31 | t.string :name 32 | end 33 | end 34 | end 35 | 36 | User.delete_all 37 | 38 | 3.times { User.create!(name: "X") } 39 | 40 | puts "Parent: #{User.first.name}" 41 | 42 | if in_worker_type == :in_ractors 43 | Parallel.each([1], in_worker_type => 1, ractor: [Callback, :call]) 44 | else 45 | Parallel.each([1], in_worker_type => 1) { |x| Callback.call x } 46 | end 47 | 48 | puts "Parent: #{User.first.name}" 49 | end 50 | -------------------------------------------------------------------------------- /spec/cases/each_with_index.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | Parallel.each_with_index(['a', 'b'], in_threads: 2) do |x, i| 5 | print "#{x}#{i}" 6 | end 7 | -------------------------------------------------------------------------------- /spec/cases/eof_in_process.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | begin 5 | Parallel.map([1]) { raise EOFError } # rubocop:disable Lint/UnreachableLoop 6 | rescue EOFError 7 | puts 'Yep, EOF' 8 | else 9 | puts 'WHOOOPS' 10 | end 11 | -------------------------------------------------------------------------------- /spec/cases/exception_raised_in_process.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | begin 5 | Parallel.each([1]) { raise StandardError } # rubocop:disable Lint/UnreachableLoop 6 | rescue Parallel::DeadWorker 7 | puts "No, DEAD worker found" 8 | rescue Exception # rubocop:disable Lint/RescueException 9 | puts "Yep, rescued the exception" 10 | else 11 | puts "WHOOOPS" 12 | end 13 | -------------------------------------------------------------------------------- /spec/cases/exit_in_process.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | begin 5 | Parallel.map([1]) { exit } # rubocop:disable Lint/UnreachableLoop 6 | rescue Parallel::DeadWorker 7 | puts "Yep, DEAD" 8 | else 9 | puts "WHOOOPS" 10 | end 11 | -------------------------------------------------------------------------------- /spec/cases/fatal_queue.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | queue = Queue.new 5 | queue.push 1 6 | queue.push 2 7 | queue.push 3 8 | Parallel.map(queue, in_threads: 2) { |(i, _id)| "ITEM-#{i}" } 9 | -------------------------------------------------------------------------------- /spec/cases/filter_map.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | result = Parallel.filter_map(['a', 'b', 'c']) do |x| 5 | x if x != 'b' 6 | end 7 | print result.inspect 8 | -------------------------------------------------------------------------------- /spec/cases/finish_in_order.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require './spec/cases/helper' 4 | 5 | class Callback 6 | def self.call(item) 7 | sleep rand * 0.01 8 | item.is_a?(Numeric) ? "F#{item}" : item 9 | end 10 | end 11 | 12 | method = ENV.fetch('METHOD') 13 | in_worker_type = :"in_#{ENV.fetch('WORKER_TYPE')}" 14 | $stdout.sync = true 15 | 16 | items = [nil, false, 2, 3, 4] 17 | finish = ->(item, index, result) { puts "finish #{item.inspect} #{index} #{result.inspect}" } 18 | options = { in_worker_type => 4, finish: finish, finish_in_order: true } 19 | if in_worker_type == :in_ractors 20 | Parallel.public_send(method, items, options.merge(ractor: [Callback, :call])) 21 | else 22 | Parallel.public_send(method, items, options) { |item| Callback.call(item) } 23 | end 24 | -------------------------------------------------------------------------------- /spec/cases/flat_map.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | result = Parallel.flat_map(['a', 'b']) do |x| 5 | [x, [x]] 6 | end 7 | print result.inspect 8 | -------------------------------------------------------------------------------- /spec/cases/helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'bundler/setup' 3 | require 'parallel' 4 | 5 | def process_diff 6 | called_from = caller(1)[0].split(":").first # forks will have the source file in their name 7 | cmd = "ps uxw|grep #{called_from}|wc -l" 8 | 9 | processes_before = `#{cmd}`.to_i 10 | 11 | yield 12 | 13 | sleep 1 14 | 15 | processes_after = `#{cmd}`.to_i 16 | 17 | if processes_before == processes_after 18 | print 'OK' 19 | else 20 | print "FAIL: before:#{processes_before} -- after:#{processes_after}" 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/cases/map_isolation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | process_diff do 5 | result = Parallel.map([1, 2, 3, 4], in_processes: 2, isolation: true) do |i| 6 | @i ||= i 7 | end 8 | puts result 9 | end 10 | -------------------------------------------------------------------------------- /spec/cases/map_with_ar.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | require "active_record" 4 | 5 | database = "parallel_with_ar_test" 6 | `mysql #{database} -e '' || mysql -e 'create database #{database}'` 7 | 8 | ActiveRecord::Schema.verbose = false 9 | ActiveRecord::Base.establish_connection( 10 | adapter: "mysql2", 11 | database: database 12 | ) 13 | 14 | class User < ActiveRecord::Base 15 | end 16 | 17 | # create tables 18 | unless User.table_exists? 19 | ActiveRecord::Schema.define(version: 1) do 20 | create_table :users do |t| 21 | t.string :name 22 | end 23 | end 24 | end 25 | 26 | User.delete_all 27 | 28 | User.create!(name: "X") 29 | 30 | Parallel.map(1..8) do |i| 31 | User.create!(name: i) 32 | end 33 | 34 | puts "User.count: #{User.count}" 35 | 36 | puts User.connection.reconnect!.inspect 37 | 38 | Parallel.map(1..8, in_threads: 4) do |i| 39 | User.create!(name: i) 40 | end 41 | 42 | User.create!(name: "X") 43 | 44 | puts User.all.map(&:name).sort.join("-") 45 | -------------------------------------------------------------------------------- /spec/cases/map_with_index.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | result = Parallel.map_with_index(['a', 'b']) do |x, i| 5 | "#{x}#{i}" 6 | end 7 | print result * '' 8 | -------------------------------------------------------------------------------- /spec/cases/map_with_index_empty.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | result = Parallel.map_with_index([]) do |x, i| 5 | "#{x}#{i}" 6 | end 7 | print result * '' 8 | -------------------------------------------------------------------------------- /spec/cases/map_with_killed_worker_before_read.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | begin 5 | Parallel.map([1, 2, 3]) do |_x, _i| 6 | Process.kill("SIGKILL", Process.pid) 7 | end 8 | rescue Parallel::DeadWorker 9 | puts "DEAD" 10 | end 11 | -------------------------------------------------------------------------------- /spec/cases/map_with_killed_worker_before_write.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | Parallel::Worker.class_eval do 5 | alias_method :work_without_kill, :work 6 | def work(*args) 7 | Process.kill("SIGKILL", pid) 8 | sleep 0.5 9 | work_without_kill(*args) 10 | end 11 | end 12 | 13 | begin 14 | Parallel.map([1, 2, 3]) do |_x, _i| 15 | Process.kill("SIGKILL", Process.pid) 16 | end 17 | rescue Parallel::DeadWorker 18 | puts "DEAD" 19 | end 20 | -------------------------------------------------------------------------------- /spec/cases/map_with_nested_arrays_and_nil.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | result = Parallel.map([1, 2, [3]]) do |x| 5 | [x, x] if x != 1 6 | end 7 | 8 | print result.inspect 9 | -------------------------------------------------------------------------------- /spec/cases/map_with_ractor.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | class Callback 5 | def self.call(arg) 6 | "#{arg}x" 7 | end 8 | end 9 | 10 | result = Parallel.map(ENV['INPUT'].chars, in_ractors: Integer(ENV["COUNT"] || 2), ractor: [Callback, :call]) 11 | print result * '' 12 | -------------------------------------------------------------------------------- /spec/cases/map_worker_number_isolation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | process_diff do 5 | result = Parallel.map([1, 2, 3, 4], in_processes: 2, isolation: true) do |_i| 6 | Parallel.worker_number 7 | end 8 | puts result.uniq.sort.join(',') 9 | end 10 | -------------------------------------------------------------------------------- /spec/cases/no_dump_with_each.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | class NotDumpable 5 | def marshal_dump 6 | raise "NOOOO" 7 | end 8 | 9 | def to_s 10 | 'not dumpable' 11 | end 12 | end 13 | 14 | Parallel.each([1]) do 15 | print 'no dump for result' 16 | NotDumpable.new 17 | end 18 | 19 | Parallel.each([NotDumpable.new]) do 20 | print 'no dump for each' 21 | 1 22 | end 23 | -------------------------------------------------------------------------------- /spec/cases/no_gc_with_each.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | Parallel.each(1..1000, in_threads: 2) do |_i| 5 | "xxxx" * 1_000_000 6 | end 7 | -------------------------------------------------------------------------------- /spec/cases/parallel_break_better_errors.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | require 'stringio' 4 | 5 | class MyException < StandardError 6 | def initialize(object) 7 | super() 8 | @object = object 9 | end 10 | end 11 | 12 | begin 13 | Parallel.in_processes(2) do 14 | ex = Parallel::Break.new 15 | # better_errors sets an instance variable that contains an array of bindings. 16 | ex.instance_variable_set :@__better_errors_bindings_stack, [ex.send(:binding)] 17 | raise ex 18 | end 19 | puts "NOTHING WAS RAISED" 20 | rescue StandardError 21 | puts $!.message 22 | puts "BACKTRACE: #{$!.backtrace.first}" 23 | end 24 | -------------------------------------------------------------------------------- /spec/cases/parallel_fast_exit.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | Parallel.map([1, 2, 3], in_processes: 2) do 5 | puts "I finished..." 6 | end 7 | 8 | sleep 10 9 | -------------------------------------------------------------------------------- /spec/cases/parallel_high_fork_rate.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | Parallel.each((0..200).to_a, in_processes: 200) do |_x| 5 | sleep 1 6 | end 7 | print 'OK' 8 | -------------------------------------------------------------------------------- /spec/cases/parallel_influence_outside_data.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | x = 'yes' 5 | 6 | Parallel.in_processes(2) do 7 | x = 'no' 8 | end 9 | print x 10 | -------------------------------------------------------------------------------- /spec/cases/parallel_kill.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | results = Parallel.map([1, 2, 3]) do |x| 5 | case x 6 | when 1 # -> stop all sub-processes, killing them instantly 7 | sleep 0.1 8 | puts "DEAD" 9 | raise Parallel::Kill 10 | when 3 11 | sleep 10 12 | else 13 | x 14 | end 15 | end 16 | 17 | puts "Works #{results.inspect}" 18 | -------------------------------------------------------------------------------- /spec/cases/parallel_map.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | result = Parallel.map(['a', 'b', 'c', 'd']) do |x| 5 | "-#{x}-" 6 | end 7 | print result * ' ' 8 | -------------------------------------------------------------------------------- /spec/cases/parallel_map_complex_objects.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | object = ["\nasd#{File.read('Gemfile')}--#{File.read('Rakefile')}" * 100, 12_345, { b: :a }] 5 | 6 | result = Parallel.map([1, 2]) do |_x| 7 | object 8 | end 9 | print 'YES' if result.inspect == [object, object].inspect 10 | -------------------------------------------------------------------------------- /spec/cases/parallel_map_range.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | result = Parallel.map(1..5) do |x| 5 | x 6 | end 7 | print result.inspect 8 | -------------------------------------------------------------------------------- /spec/cases/parallel_map_sleeping.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | Parallel.map(['a', 'b', 'c', 'd']) do |_x| 5 | sleep 1 6 | end 7 | -------------------------------------------------------------------------------- /spec/cases/parallel_map_uneven.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | Parallel.map([1, 2, 1, 2]) do |x| 5 | sleep 2 if x == 1 6 | end 7 | -------------------------------------------------------------------------------- /spec/cases/parallel_raise.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | begin 5 | Parallel.in_processes(2) do 6 | raise "TEST" 7 | end 8 | puts "FAIL" 9 | rescue RuntimeError 10 | puts $!.message 11 | end 12 | -------------------------------------------------------------------------------- /spec/cases/parallel_raise_undumpable.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | require 'stringio' 4 | 5 | class MyException < StandardError 6 | def initialize(object) 7 | super() 8 | @object = object 9 | end 10 | end 11 | 12 | begin 13 | Parallel.in_processes(2) do 14 | raise MyException, StringIO.new 15 | end 16 | puts "NOTHING WAS RAISED" 17 | rescue StandardError 18 | puts $!.message 19 | puts "BACKTRACE: #{$!.backtrace.first}" 20 | end 21 | -------------------------------------------------------------------------------- /spec/cases/parallel_sleeping_2.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | Parallel.in_processes(5) do 5 | sleep 2 6 | end 7 | -------------------------------------------------------------------------------- /spec/cases/parallel_start_and_kill.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | method = case ARGV[0] 5 | when "PROCESS" then :in_processes 6 | when "THREAD" then :in_threads 7 | else raise "Learn to use this!" 8 | end 9 | 10 | options = {} 11 | options[:count] = 2 12 | if ARGV.length > 1 13 | options[:interrupt_signal] = ARGV[1].to_s 14 | trap('SIGINT') { puts 'Wrapper caught SIGINT' } if ARGV[1] != 'SIGINT' 15 | end 16 | 17 | Parallel.send(method, options) do 18 | sleep 5 19 | puts "I should have been killed earlier..." 20 | end 21 | -------------------------------------------------------------------------------- /spec/cases/parallel_with_detected_cpus.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | x = Parallel.in_processes do 5 | "HELLO" 6 | end 7 | puts x 8 | -------------------------------------------------------------------------------- /spec/cases/parallel_with_nil_uses_detected_cpus.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | x = Parallel.in_processes(nil) do 5 | "HELLO" 6 | end 7 | puts x 8 | -------------------------------------------------------------------------------- /spec/cases/parallel_with_set_processes.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | x = Parallel.in_processes(5) do 5 | "HELLO" 6 | end 7 | puts x 8 | -------------------------------------------------------------------------------- /spec/cases/profile_memory.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | def count_objects 3 | old = Hash.new(0) 4 | cur = Hash.new(0) 5 | GC.start 6 | ObjectSpace.each_object { |o| old[o.class] += 1 } 7 | yield 8 | GC.start 9 | GC.start 10 | ObjectSpace.each_object { |o| cur[o.class] += 1 } 11 | cur.to_h { |k, v| [k, v - old[k]] }.reject { |_k, v| v == 0 } 12 | end 13 | 14 | class Callback 15 | def self.call(x); end 16 | end 17 | 18 | require './spec/cases/helper' 19 | 20 | items = Array.new(1000) 21 | options = { "in_#{ARGV[0]}": 2 } 22 | 23 | # TODO: not sure why this fails without 2.times in threading mode :( 24 | 25 | call = lambda do 26 | if ARGV[0] == "ractors" 27 | Parallel.map(items, options.merge(ractor: [Callback, :call])) 28 | sleep 0.1 # ractors need a bit to shut down 29 | else 30 | Parallel.map(items, options) {} 31 | end 32 | end 33 | 34 | puts(count_objects { 2.times { call.call } }.inspect) 35 | 36 | puts(count_objects { 2.times { call.call } }.inspect) 37 | -------------------------------------------------------------------------------- /spec/cases/progress.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | title = (ENV["TITLE"] == "true" ? true : "Doing stuff") 5 | Parallel.map(1..50, progress: title) do 6 | sleep 1 if $stdout.tty? # for debugging 7 | end 8 | -------------------------------------------------------------------------------- /spec/cases/progress_with_finish.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | sum = 0 5 | finish = ->(_item, _index, result) { sum += result } 6 | 7 | Parallel.map(1..50, progress: "Doing stuff", finish: finish) do 8 | sleep 1 if $stdout.tty? # for debugging 9 | 2 10 | end 11 | 12 | puts sum 13 | -------------------------------------------------------------------------------- /spec/cases/progress_with_options.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | # ruby-progressbar ignores the format string you give it 5 | # unless the output is a TTY. When running in the test, 6 | # the output is not a TTY, so we cannot test that the format 7 | # string you pass overrides parallel's default. So, we pretend 8 | # that stdout is a TTY to test that the options are merged 9 | # in the correct way. 10 | tty_stdout = $stdout 11 | class << tty_stdout 12 | def tty? 13 | true 14 | end 15 | end 16 | 17 | parallel_options = { 18 | progress: { 19 | title: "Reticulating Splines", 20 | progress_mark: ';', 21 | format: "%t %w", 22 | output: tty_stdout 23 | } 24 | } 25 | 26 | Parallel.map(1..50, parallel_options) do 27 | 2 28 | end 29 | -------------------------------------------------------------------------------- /spec/cases/synchronizes_start_and_finish.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | start = lambda { |item, _index| 5 | print item * 5 6 | sleep rand * 0.2 7 | puts item * 5 8 | } 9 | finish = lambda { |_item, _index, result| 10 | print result * 5 11 | sleep rand * 0.2 12 | puts result * 5 13 | } 14 | Parallel.map(['a', 'b', 'c'], start: start, finish: finish) do |i| 15 | sleep rand * 0.2 16 | i.upcase 17 | end 18 | -------------------------------------------------------------------------------- /spec/cases/timeout_in_threads.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | require 'timeout' 4 | 5 | Parallel.each([1], in_threads: 1) do |_i| 6 | Timeout.timeout(0.1) { sleep 0.2 } 7 | rescue Timeout::Error 8 | puts "OK" 9 | else 10 | puts "BROKEN" 11 | end 12 | -------------------------------------------------------------------------------- /spec/cases/with_break.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | $stdout.sync = true # otherwise results can go weird... 4 | 5 | method = ENV.fetch('METHOD') 6 | in_worker_type = :"in_#{ENV.fetch('WORKER_TYPE')}" 7 | worker_size = (ENV['WORKER_SIZE'] || 4).to_i 8 | 9 | ARGV.freeze # make ractor happy 10 | 11 | class Callback 12 | def self.call(x) 13 | $stdout.sync = true 14 | sleep 0.1 # so all workers get started 15 | print x 16 | raise Parallel::Break, *ARGV if x == 1 17 | sleep 0.2 # so now no work gets queued before Parallel::Break is raised 18 | x 19 | end 20 | end 21 | 22 | options = { in_worker_type => worker_size } 23 | result = 24 | if in_worker_type == :in_ractors 25 | Parallel.public_send(method, 1..10, options.merge(ractor: [Callback, :call])) 26 | else 27 | Parallel.public_send(method, 1..10, options) { |x| Callback.call x } 28 | end 29 | print " Parallel::Break raised - result #{result.inspect}" 30 | -------------------------------------------------------------------------------- /spec/cases/with_break_before_finish.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | method = ENV.fetch('METHOD') 5 | in_worker_type = :"in_#{ENV.fetch('WORKER_TYPE')}" 6 | $stdout.sync = true 7 | 8 | class Callback 9 | def self.call(x) 10 | $stdout.sync = true 11 | sleep 0.1 # let workers start 12 | raise Parallel::Break if x == 1 13 | sleep 0.2 14 | print x 15 | x 16 | end 17 | end 18 | 19 | finish = lambda do |_item, _index, _result| 20 | sleep 0.1 21 | print "finish hook called" 22 | end 23 | 24 | options = { in_worker_type => 4, finish: finish } 25 | if in_worker_type == :in_ractors 26 | Parallel.public_send(method, 1..10, options.merge(ractor: [Callback, :call])) 27 | else 28 | Parallel.public_send(method, 1..10, options) { |x| Callback.call x } 29 | end 30 | print " Parallel::Break raised" 31 | -------------------------------------------------------------------------------- /spec/cases/with_exception.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | $stdout.sync = true # otherwise results can go weird... 4 | 5 | method = ENV.fetch('METHOD') 6 | in_worker_type = :"in_#{ENV.fetch('WORKER_TYPE')}" 7 | worker_size = (ENV['WORKER_SIZE'] || 4).to_i 8 | 9 | class ParallelTestError < StandardError 10 | end 11 | 12 | class Callback 13 | def self.call(x) 14 | $stdout.sync = true 15 | sleep 0.1 # so all workers get started 16 | print x 17 | raise ParallelTestError, 'foo' if x == 1 18 | sleep 0.2 # so now no work gets queued before exception is raised 19 | x 20 | end 21 | end 22 | 23 | begin 24 | options = { in_worker_type => worker_size } 25 | if in_worker_type == :in_ractors 26 | Parallel.public_send(method, 1..100, options.merge(ractor: [Callback, :call])) 27 | else 28 | Parallel.public_send(method, 1..100, options) { |x| Callback.call x } 29 | end 30 | rescue ParallelTestError 31 | print ' raised' 32 | end 33 | -------------------------------------------------------------------------------- /spec/cases/with_exception_before_finish.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | method = ENV.fetch('METHOD') 5 | in_worker_type = :"in_#{ENV.fetch('WORKER_TYPE')}" 6 | 7 | class ParallelTestError < StandardError 8 | end 9 | 10 | class Callback 11 | def self.call(x) 12 | $stdout.sync = true 13 | if x != 3 14 | sleep 0.2 15 | raise ParallelTestError 16 | end 17 | print x 18 | x 19 | end 20 | end 21 | 22 | begin 23 | finish = lambda do |_item, _index, _result| 24 | print " called" 25 | end 26 | 27 | options = { in_worker_type => 4, finish: finish } 28 | if in_worker_type == :in_ractors 29 | Parallel.public_send(method, 1..10, options.merge(ractor: [Callback, :call])) 30 | else 31 | Parallel.public_send(method, 1..10, options) { |x| Callback.call x } 32 | end 33 | rescue ParallelTestError 34 | nil 35 | end 36 | -------------------------------------------------------------------------------- /spec/cases/with_exception_in_finish.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | $stdout.sync = true 5 | method = ENV.fetch('METHOD') 6 | in_worker_type = :"in_#{ENV.fetch('WORKER_TYPE')}" 7 | 8 | class ParallelTestError < StandardError 9 | end 10 | 11 | class Callback 12 | def self.call(x) 13 | $stdout.sync = true 14 | print x 15 | sleep 0.2 # let everyone start and print 16 | sleep 0.2 unless x == 1 # prevent other work from start/finish before exception 17 | x 18 | end 19 | end 20 | 21 | begin 22 | finish = lambda do |x, _index, _result| 23 | raise ParallelTestError, 'foo' if x == 1 24 | end 25 | options = { in_worker_type => 4, finish: finish } 26 | if in_worker_type == :in_ractors 27 | Parallel.public_send(method, 1..10, options.merge(ractor: [Callback, :call])) 28 | else 29 | Parallel.public_send(method, 1..10, options) { |x| Callback.call x } 30 | end 31 | rescue ParallelTestError 32 | print ' raised' 33 | end 34 | -------------------------------------------------------------------------------- /spec/cases/with_exception_in_start.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | method = ENV.fetch('METHOD') 5 | in_worker_type = :"in_#{ENV.fetch('WORKER_TYPE')}" 6 | 7 | class ParallelTestError < StandardError 8 | end 9 | 10 | class Callback 11 | def self.call(x) 12 | $stdout.sync = true 13 | print x 14 | sleep 0.2 # so now no work gets queued before exception is raised 15 | x 16 | end 17 | end 18 | 19 | begin 20 | start = lambda do |_item, _index| 21 | @started = (@started ? @started + 1 : 1) 22 | sleep 0.01 # a bit of time for ractors to work 23 | raise ParallelTestError, 'foo' if @started == 4 24 | end 25 | 26 | options = { in_worker_type => 4, start: start } 27 | if in_worker_type == :in_ractors 28 | Parallel.public_send(method, 1..10, options.merge(ractor: [Callback, :call])) 29 | else 30 | Parallel.public_send(method, 1..10, options) { |x| Callback.call x } 31 | end 32 | rescue ParallelTestError 33 | print ' raised' 34 | end 35 | -------------------------------------------------------------------------------- /spec/cases/with_exception_in_start_before_finish.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | method = ENV.fetch('METHOD') 5 | in_worker_type = :"in_#{ENV.fetch('WORKER_TYPE')}" 6 | $stdout.sync = true 7 | 8 | class ParallelTestError < StandardError 9 | end 10 | 11 | class Callback 12 | def self.call(x) 13 | $stdout.sync = true 14 | puts "call #{x}" 15 | x 16 | end 17 | end 18 | 19 | begin 20 | start = lambda do |item, index| 21 | puts "start #{index}" 22 | if item != 3 23 | sleep 0.2 24 | raise ParallelTestError 25 | end 26 | end 27 | 28 | finish = lambda do |_item, index, _result| 29 | puts "finish #{index}" 30 | end 31 | 32 | options = { in_worker_type => 4, start: start, finish: finish } 33 | 34 | if in_worker_type == :in_ractors 35 | Parallel.public_send(method, 1..10, options.merge(ractor: [Callback, :call])) 36 | else 37 | Parallel.public_send(method, 1..10, options) { |x| Callback.call x } 38 | end 39 | rescue ParallelTestError 40 | nil 41 | end 42 | -------------------------------------------------------------------------------- /spec/cases/with_lambda.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | $stdout.sync = true 5 | type = :"in_#{ARGV.fetch(0)}" 6 | all = [3, 2, 1] 7 | produce = -> { all.pop || Parallel::Stop } 8 | 9 | class Callback 10 | def self.call(x) 11 | $stdout.sync = true 12 | "ITEM-#{x}" 13 | end 14 | end 15 | 16 | if type == :in_ractors 17 | puts(Parallel.map(produce, type => 2, ractor: [Callback, :call])) 18 | else 19 | puts(Parallel.map(produce, type => 2) { |(i, _id)| Callback.call i }) 20 | end 21 | -------------------------------------------------------------------------------- /spec/cases/with_queue.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | type = :"in_#{ARGV.fetch(0)}" 5 | 6 | class Callback 7 | def self.call(x) 8 | "ITEM-#{x}" 9 | end 10 | end 11 | 12 | queue = Queue.new 13 | Thread.new do 14 | sleep 0.2 15 | queue.push 1 16 | queue.push 2 17 | queue.push 3 18 | queue.push Parallel::Stop 19 | end 20 | 21 | if type == :in_ractors 22 | puts(Parallel.map(queue, type => 2, ractor: [Callback, :call])) 23 | else 24 | puts(Parallel.map(queue, type => 2) { |(i, _id)| Callback.call i }) 25 | end 26 | -------------------------------------------------------------------------------- /spec/cases/with_worker_number.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require './spec/cases/helper' 3 | 4 | method = ENV.fetch('METHOD') 5 | in_worker_type = :"in_#{ENV.fetch('WORKER_TYPE')}" 6 | 7 | Parallel.public_send(method, 1..100, in_worker_type => 4) do 8 | sleep 0.1 # so all workers get started 9 | print Parallel.worker_number 10 | end 11 | -------------------------------------------------------------------------------- /spec/parallel_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'spec_helper' 3 | 4 | describe Parallel do 5 | worker_types = ["threads"] 6 | worker_types << "processes" if Process.respond_to?(:fork) 7 | worker_types << "ractors" if defined?(Ractor) 8 | 9 | def time_taken 10 | t = Time.now.to_f 11 | yield 12 | RUBY_ENGINE == "jruby" ? 0 : Time.now.to_f - t # jruby is super slow ... don't blow up all the tests ... 13 | end 14 | 15 | def kill_process_with_name(file, signal = 'INT') 16 | running_processes = `ps -f`.split("\n").map { |line| line.split(/\s+/) } 17 | pid_index = running_processes.detect { |p| p.include?("UID") }.index("UID") + 1 18 | parent_pid = running_processes.detect { |p| p.include?(file) and !p.include?("sh") }[pid_index] 19 | `kill -s #{signal} #{parent_pid}` 20 | end 21 | 22 | def execute_start_and_kill(command, amount, signal = 'INT') 23 | t = nil 24 | lambda { 25 | t = Thread.new { ruby("spec/cases/parallel_start_and_kill.rb #{command} 2>&1 && echo 'FINISHED'") } 26 | sleep 1.5 27 | kill_process_with_name('spec/cases/parallel_start_and_kill.rb', signal) 28 | sleep 1 29 | }.should change { `ps`.split("\n").size }.by amount 30 | t.value 31 | end 32 | 33 | def without_ractor_warning(out) 34 | out.sub(/.*Ractor is experimental.*\n/, "") 35 | end 36 | 37 | describe ".processor_count" do 38 | before do 39 | Parallel.instance_variable_set(:@processor_count, nil) 40 | end 41 | 42 | it "returns a number" do 43 | (1..999).should include(Parallel.processor_count) 44 | end 45 | 46 | if RUBY_PLATFORM =~ /darwin10/ 47 | it 'works if hwprefs in not available' do 48 | Parallel.should_receive(:hwprefs_available?).and_return false 49 | (1..999).should include(Parallel.processor_count) 50 | end 51 | end 52 | end 53 | 54 | describe ".physical_processor_count" do 55 | before do 56 | Parallel.instance_variable_set(:@physical_processor_count, nil) 57 | end 58 | 59 | it "returns a number" do 60 | (1..999).should include(Parallel.physical_processor_count) 61 | end 62 | 63 | it "is even factor of logical cpus" do 64 | (Parallel.processor_count % Parallel.physical_processor_count).should == 0 65 | end 66 | end 67 | 68 | describe ".in_processes" do 69 | def cpus 70 | Parallel.processor_count 71 | end 72 | 73 | it "executes with detected cpus" do 74 | ruby("spec/cases/parallel_with_detected_cpus.rb").should == "HELLO\n" * cpus 75 | end 76 | 77 | it "executes with detected cpus when nil was given" do 78 | ruby("spec/cases/parallel_with_nil_uses_detected_cpus.rb").should == "HELLO\n" * cpus 79 | end 80 | 81 | it "executes with cpus from ENV" do 82 | `PARALLEL_PROCESSOR_COUNT=10 ruby spec/cases/parallel_with_detected_cpus.rb`.should == "HELLO\n" * 10 83 | end 84 | 85 | it "set amount of parallel processes" do 86 | ruby("spec/cases/parallel_with_set_processes.rb").should == "HELLO\n" * 5 87 | end 88 | 89 | it "enforces only one worker type" do 90 | -> { Parallel.map([1, 2, 3], in_processes: 2, in_threads: 3) }.should raise_error(ArgumentError) 91 | end 92 | 93 | it "does not influence outside data" do 94 | ruby("spec/cases/parallel_influence_outside_data.rb").should == "yes" 95 | end 96 | 97 | it "kills the processes when the main process gets killed through ctrl+c" do 98 | time_taken do 99 | result = execute_start_and_kill "PROCESS", 0 100 | result.should_not include "FINISHED" 101 | end.should be <= 3 102 | end 103 | 104 | it "kills the processes when the main process gets killed through a custom interrupt" do 105 | time_taken do 106 | execute_start_and_kill "PROCESS SIGTERM", 0, "TERM" 107 | end.should be <= 3 108 | end 109 | 110 | it "kills the threads when the main process gets killed through ctrl+c" do 111 | time_taken do 112 | result = execute_start_and_kill "THREAD", 0 113 | result.should_not include "FINISHED" 114 | end.should be <= 3 115 | end 116 | 117 | it "does not kill processes when the main process gets sent an interrupt besides the custom interrupt" do 118 | time_taken do 119 | result = execute_start_and_kill "PROCESS SIGTERM", 4 120 | result.should include 'FINISHED' 121 | result.should include 'Wrapper caught SIGINT' 122 | result.should include 'I should have been killed earlier' 123 | end.should be <= 7 124 | end 125 | 126 | it "does not kill threads when the main process gets sent an interrupt besides the custom interrupt" do 127 | time_taken do 128 | result = execute_start_and_kill "THREAD SIGTERM", 2 129 | result.should include 'FINISHED' 130 | result.should include 'Wrapper caught SIGINT' 131 | result.should include 'I should have been killed earlier' 132 | end.should be <= 7 133 | end 134 | 135 | it "does not kill anything on ctrl+c when everything has finished" do 136 | time_taken do 137 | t = Thread.new { ruby("spec/cases/parallel_fast_exit.rb 2>&1") } 138 | sleep 2 139 | kill_process_with_name("spec/cases/parallel_fast_exit.rb") # simulates Ctrl+c 140 | sleep 1 141 | result = t.value 142 | result.scan("I finished").size.should == 3 143 | result.should_not include("Parallel execution interrupted") 144 | end.should <= 4 145 | end 146 | 147 | it "preserves original intrrupts" do 148 | t = Thread.new { ruby("spec/cases/double_interrupt.rb 2>&1 && echo FIN") } 149 | sleep 2 150 | kill_process_with_name("spec/cases/double_interrupt.rb") # simulates Ctrl+c 151 | sleep 1 152 | result = t.value 153 | result.should include("YES") 154 | result.should include("FIN") 155 | end 156 | 157 | it "restores original intrrupts" do 158 | ruby("spec/cases/after_interrupt.rb 2>&1").should == "DEFAULT\n" 159 | end 160 | 161 | it "saves time" do 162 | time_taken do 163 | ruby("spec/cases/parallel_sleeping_2.rb") 164 | end.should < 3.5 165 | end 166 | 167 | it "raises when one of the processes raises" do 168 | ruby("spec/cases/parallel_raise.rb").strip.should == 'TEST' 169 | end 170 | 171 | it "can raise an undumpable exception" do 172 | out = ruby("spec/cases/parallel_raise_undumpable.rb").strip 173 | out.sub!(Dir.pwd, '.') # relative paths 174 | out.gsub!(/(\d+):.*/, "\\1") # no diff in ruby version xyz.rb:123:in `block in
' 175 | out.should == "MyException: MyException\nBACKTRACE: spec/cases/parallel_raise_undumpable.rb:14" 176 | end 177 | 178 | it "can handle Break exceptions when the better_errors gem is installed" do 179 | out = ruby("spec/cases/parallel_break_better_errors.rb").strip 180 | out.should == "NOTHING WAS RAISED" 181 | end 182 | 183 | it 'can handle to high fork rate' do 184 | next if RbConfig::CONFIG["target_os"].include?("darwin1") # kills macs for some reason 185 | ruby("spec/cases/parallel_high_fork_rate.rb").should == 'OK' 186 | end 187 | 188 | it 'does not leave processes behind while running' do 189 | ruby("spec/cases/closes_processes_at_runtime.rb").gsub(/.* deprecated; use BigDecimal.*\n/, '').should == 'OK' 190 | end 191 | 192 | it "does not open unnecessary pipes" do 193 | max = (RbConfig::CONFIG["target_os"].include?("darwin1") ? 10 : 1800) # somehow super bad on CI 194 | ruby("spec/cases/count_open_pipes.rb").to_i.should < max 195 | end 196 | end 197 | 198 | describe ".in_threads" do 199 | it "saves time" do 200 | time_taken do 201 | Parallel.in_threads(3) { sleep 2 } 202 | end.should < 3 203 | end 204 | 205 | it "does not create new processes" do 206 | -> { Thread.new { Parallel.in_threads(2) { sleep 1 } } }.should_not(change { `ps`.split("\n").size }) 207 | end 208 | 209 | it "returns results as array" do 210 | Parallel.in_threads(4) { |i| "XXX#{i}" }.should == ["XXX0", 'XXX1', 'XXX2', 'XXX3'] 211 | end 212 | 213 | it "raises when a thread raises" do 214 | Thread.report_on_exception = false 215 | -> { Parallel.in_threads(2) { |_i| raise "TEST" } }.should raise_error("TEST") 216 | ensure 217 | Thread.report_on_exception = true 218 | end 219 | end 220 | 221 | describe ".map" do 222 | it "saves time" do 223 | time_taken do 224 | ruby("spec/cases/parallel_map_sleeping.rb") 225 | end.should <= 3.5 226 | end 227 | 228 | it "does not modify options" do 229 | -> { Parallel.map([], {}.freeze) }.should_not raise_error 230 | end 231 | 232 | it "executes with given parameters" do 233 | ruby("spec/cases/parallel_map.rb").should == "-a- -b- -c- -d-" 234 | end 235 | 236 | it "can dump/load complex objects" do 237 | ruby("spec/cases/parallel_map_complex_objects.rb").should == "YES" 238 | end 239 | 240 | it "starts new process immediately when old exists" do 241 | time_taken do 242 | ruby("spec/cases/parallel_map_uneven.rb") 243 | end.should <= 3.5 244 | end 245 | 246 | it "does not flatten results" do 247 | Parallel.map([1, 2, 3], in_threads: 2) { |x| [x, x] }.should == [[1, 1], [2, 2], [3, 3]] 248 | end 249 | 250 | it "can run in threads" do 251 | result = Parallel.map([1, 2, 3, 4, 5, 6, 7, 8, 9], in_threads: 4) { |x| x + 2 } 252 | result.should == [3, 4, 5, 6, 7, 8, 9, 10, 11] 253 | end 254 | 255 | it 'supports all Enumerable-s' do 256 | ruby("spec/cases/parallel_map_range.rb").should == '[1, 2, 3, 4, 5]' 257 | end 258 | 259 | it 'handles nested arrays and nil correctly' do 260 | ruby("spec/cases/map_with_nested_arrays_and_nil.rb").should == '[nil, [2, 2], [[3], [3]]]' 261 | end 262 | 263 | worker_types.each do |type| 264 | it "does not queue new work when one fails in #{type}" do 265 | out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception.rb 2>&1` 266 | without_ractor_warning(out).should =~ /\A\d{4} raised\z/ 267 | end 268 | 269 | it "does not queue new work when one raises Break in #{type}" do 270 | out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_break.rb 2>&1` 271 | without_ractor_warning(out).should =~ /\A\d{4} Parallel::Break raised - result nil\z/ 272 | end 273 | 274 | it "stops all workers when a start hook fails with #{type}" do 275 | out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start.rb 2>&1` 276 | out = without_ractor_warning(out) 277 | if type == "ractors" 278 | # TODO: running ractors should be interrupted 279 | out.should =~ /\A.*raised.*\z/ 280 | out.should_not =~ /5/ # stopped at 4 281 | else 282 | out.should =~ /\A\d{3} raised\z/ 283 | end 284 | end 285 | 286 | it "does not add new work when a finish hook fails with #{type}" do 287 | out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_finish.rb 2>&1` 288 | without_ractor_warning(out).should =~ /\A\d{4} raised\z/ 289 | end 290 | 291 | it "does not call the finish hook when a worker fails with #{type}" do 292 | out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_before_finish.rb 2>&1` 293 | without_ractor_warning(out).should == '3 called' 294 | end 295 | 296 | it "does not call the finish hook when a worker raises Break in #{type}" do 297 | out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_break_before_finish.rb 2>&1` 298 | without_ractor_warning(out).should =~ /\A\d{3}(finish hook called){3} Parallel::Break raised\z/ 299 | end 300 | 301 | it "does not call the finish hook when a start hook fails with #{type}" do 302 | out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start_before_finish.rb 2>&1` 303 | if type == "ractors" 304 | # we are calling on the main thread, so everything sleeps 305 | without_ractor_warning(out).should == "start 0\n" 306 | else 307 | out.split("\n").sort.join("\n").should == <<~OUT.rstrip 308 | call 3 309 | finish 2 310 | start 0 311 | start 1 312 | start 2 313 | start 3 314 | OUT 315 | end 316 | end 317 | 318 | it "can return from break with #{type}" do 319 | out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_break.rb hi 2>&1` 320 | out.should =~ /^\d{4} Parallel::Break raised - result "hi"$/ 321 | end 322 | 323 | it "sets Parallel.worker_number with 4 #{type}" do 324 | skip if type == "ractors" # not supported 325 | out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_worker_number.rb 2>&1` 326 | out.should =~ /\A[0123]+\z/ 327 | ['0', '1', '2', '3'].each { |number| out.should include number } 328 | end 329 | 330 | it "sets Parallel.worker_number with 0 #{type}" do 331 | skip if type == "ractors" # not supported 332 | type_key = :"in_#{type}" 333 | result = Parallel.map([1, 2, 3, 4, 5, 6, 7, 8, 9], type_key => 0) { |_x| Parallel.worker_number } 334 | result.uniq.should == [0] 335 | Parallel.worker_number.should be_nil 336 | end 337 | 338 | it "can run with 0 by not using #{type}" do 339 | Thread.should_not_receive(:exclusive) 340 | Process.should_not_receive(:fork) 341 | result = Parallel.map([1, 2, 3, 4, 5, 6, 7, 8, 9], "in_#{type}": 0) { |x| x + 2 } 342 | result.should == [3, 4, 5, 6, 7, 8, 9, 10, 11] 343 | end 344 | 345 | it "can call finish hook in order #{type}" do 346 | out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/finish_in_order.rb 2>&1` 347 | without_ractor_warning(out).should == <<~OUT 348 | finish nil 0 nil 349 | finish false 1 false 350 | finish 2 2 "F2" 351 | finish 3 3 "F3" 352 | finish 4 4 "F4" 353 | OUT 354 | end 355 | end 356 | 357 | it "notifies when an item of work is dispatched to a worker process" do 358 | monitor = double('monitor', call: nil) 359 | monitor.should_receive(:call).once.with(:first, 0) 360 | monitor.should_receive(:call).once.with(:second, 1) 361 | monitor.should_receive(:call).once.with(:third, 2) 362 | Parallel.map([:first, :second, :third], start: monitor, in_processes: 3) {} 363 | end 364 | 365 | it "notifies when an item of work is dispatched with 0 processes" do 366 | monitor = double('monitor', call: nil) 367 | monitor.should_receive(:call).once.with(:first, 0) 368 | monitor.should_receive(:call).once.with(:second, 1) 369 | monitor.should_receive(:call).once.with(:third, 2) 370 | Parallel.map([:first, :second, :third], start: monitor, in_processes: 0) {} 371 | end 372 | 373 | it "notifies when an item of work is completed by a worker process" do 374 | monitor = double('monitor', call: nil) 375 | monitor.should_receive(:call).once.with(:first, 0, 123) 376 | monitor.should_receive(:call).once.with(:second, 1, 123) 377 | monitor.should_receive(:call).once.with(:third, 2, 123) 378 | Parallel.map([:first, :second, :third], finish: monitor, in_processes: 3) { 123 } 379 | end 380 | 381 | it "notifies when an item of work is completed with 0 processes" do 382 | monitor = double('monitor', call: nil) 383 | monitor.should_receive(:call).once.with(:first, 0, 123) 384 | monitor.should_receive(:call).once.with(:second, 1, 123) 385 | monitor.should_receive(:call).once.with(:third, 2, 123) 386 | Parallel.map([:first, :second, :third], finish: monitor, in_processes: 0) { 123 } 387 | end 388 | 389 | it "notifies when an item of work is dispatched to a threaded worker" do 390 | monitor = double('monitor', call: nil) 391 | monitor.should_receive(:call).once.with(:first, 0) 392 | monitor.should_receive(:call).once.with(:second, 1) 393 | monitor.should_receive(:call).once.with(:third, 2) 394 | Parallel.map([:first, :second, :third], start: monitor, in_threads: 3) {} 395 | end 396 | 397 | it "notifies when an item of work is dispatched with 0 threads" do 398 | monitor = double('monitor', call: nil) 399 | monitor.should_receive(:call).once.with(:first, 0) 400 | monitor.should_receive(:call).once.with(:second, 1) 401 | monitor.should_receive(:call).once.with(:third, 2) 402 | Parallel.map([:first, :second, :third], start: monitor, in_threads: 0) {} 403 | end 404 | 405 | it "notifies when an item of work is completed by a threaded worker" do 406 | monitor = double('monitor', call: nil) 407 | monitor.should_receive(:call).once.with(:first, 0, 123) 408 | monitor.should_receive(:call).once.with(:second, 1, 123) 409 | monitor.should_receive(:call).once.with(:third, 2, 123) 410 | Parallel.map([:first, :second, :third], finish: monitor, in_threads: 3) { 123 } 411 | end 412 | 413 | it "notifies when an item of work is completed with 0 threads" do 414 | monitor = double('monitor', call: nil) 415 | monitor.should_receive(:call).once.with(:first, 0, 123) 416 | monitor.should_receive(:call).once.with(:second, 1, 123) 417 | monitor.should_receive(:call).once.with(:third, 2, 123) 418 | Parallel.map([:first, :second, :third], finish: monitor, in_threads: 0) { 123 } 419 | end 420 | 421 | it "spits out a useful error when a worker dies before read" do 422 | ruby("spec/cases/map_with_killed_worker_before_read.rb 2>&1").should include "DEAD" 423 | end 424 | 425 | it "spits out a useful error when a worker dies before write" do 426 | ruby("spec/cases/map_with_killed_worker_before_write.rb 2>&1").should include "DEAD" 427 | end 428 | 429 | it "raises DeadWorker when using exit so people learn to not kill workers and do not crash main process" do 430 | ruby("spec/cases/exit_in_process.rb 2>&1").should include "Yep, DEAD" 431 | end 432 | 433 | it "rescues the Exception raised in child process" do 434 | ruby("spec/cases/exception_raised_in_process.rb 2>&1").should include "Yep, rescued the exception" 435 | end 436 | 437 | it 'raises EOF (not DeadWorker) when a worker raises EOF in process' do 438 | ruby("spec/cases/eof_in_process.rb 2>&1").should include 'Yep, EOF' 439 | end 440 | 441 | it "threads can be killed instantly" do 442 | mutex = Mutex.new 443 | state = [nil, nil] 444 | children = [nil, nil] 445 | thread = Thread.new do 446 | parent = Thread.current 447 | Parallel.map([0, 1], in_threads: 2) do |i| 448 | mutex.synchronize { children[i] = Thread.current } 449 | mutex.synchronize { state[i] = :ready } 450 | parent.join 451 | mutex.synchronize { state[i] = :error } 452 | end 453 | end 454 | Thread.pass while state.any?(&:nil?) 455 | thread.kill 456 | Thread.pass while children.any?(&:alive?) 457 | state[0].should == :ready 458 | state[1].should == :ready 459 | end 460 | 461 | it "processes can be killed instantly" do 462 | pipes = [IO.pipe, IO.pipe] 463 | thread = Thread.new do 464 | Parallel.map([0, 1, 2, 3], in_processes: 2) do |i| 465 | pipes[i % 2][0].close unless pipes[i % 2][0].closed? 466 | Marshal.dump('finish', pipes[i % 2][1]) 467 | sleep 1 468 | nil 469 | end 470 | end 471 | [0, 1].each do |i| 472 | Marshal.load(pipes[i][0]).should == 'finish' 473 | end 474 | pipes.each { |pipe| pipe[1].close } 475 | thread.kill 476 | pipes.each do |pipe| 477 | begin 478 | ret = Marshal.load(pipe[0]) 479 | rescue EOFError 480 | ret = :error 481 | end 482 | ret.should == :error 483 | end 484 | pipes.each { |pipe| pipe[0].close } 485 | end 486 | 487 | it "synchronizes :start and :finish" do 488 | out = ruby("spec/cases/synchronizes_start_and_finish.rb") 489 | ['a', 'b', 'c'].each do |letter| 490 | out.sub! letter.downcase * 10, 'OK' 491 | out.sub! letter.upcase * 10, 'OK' 492 | end 493 | out.should == "OK\n" * 6 494 | end 495 | 496 | it 'is equivalent to serial map' do 497 | l = Array.new(10_000) { |i| i } 498 | Parallel.map(l, { in_threads: 4 }) { |x| x + 1 }.should == l.map { |x| x + 1 } 499 | end 500 | 501 | it 'can work in isolation' do 502 | out = ruby("spec/cases/map_isolation.rb") 503 | out.should == "1\n2\n3\n4\nOK" 504 | end 505 | 506 | it 'sets Parallel.worker_number when run with isolation' do 507 | out = ruby("spec/cases/map_worker_number_isolation.rb") 508 | out.should == "0,1\nOK" 509 | end 510 | 511 | it 'can use Timeout' do 512 | out = ruby("spec/cases/timeout_in_threads.rb") 513 | out.should == "OK\n" 514 | end 515 | end 516 | 517 | describe ".map_with_index" do 518 | it "yields object and index" do 519 | ruby("spec/cases/map_with_index.rb 2>&1").should == 'a0b1' 520 | end 521 | 522 | it "does not crash with empty set" do 523 | ruby("spec/cases/map_with_index_empty.rb 2>&1").should == '' 524 | end 525 | 526 | it "can run with 0 threads" do 527 | Thread.should_not_receive(:exclusive) 528 | Parallel.map_with_index([1, 2, 3, 4, 5, 6, 7, 8, 9], in_threads: 0) do |x, _i| 529 | x + 2 530 | end.should == [3, 4, 5, 6, 7, 8, 9, 10, 11] 531 | end 532 | 533 | it "can run with 0 processes" do 534 | Process.should_not_receive(:fork) 535 | Parallel.map_with_index([1, 2, 3, 4, 5, 6, 7, 8, 9], in_processes: 0) do |x, _i| 536 | x + 2 537 | end.should == [3, 4, 5, 6, 7, 8, 9, 10, 11] 538 | end 539 | end 540 | 541 | describe ".flat_map" do 542 | it "yields object and index" do 543 | ruby("spec/cases/flat_map.rb 2>&1").should == '["a", ["a"], "b", ["b"]]' 544 | end 545 | end 546 | 547 | describe ".filter_map" do 548 | it "yields object" do 549 | ruby("spec/cases/filter_map.rb 2>&1").should == '["a", "c"]' 550 | end 551 | end 552 | 553 | describe ".any?" do 554 | it "returns true if any result is truthy" do 555 | ruby("spec/cases/any_true.rb").split(',').should == ['true'] * 3 * 2 556 | end 557 | 558 | it "returns false if all results are falsy" do 559 | ruby("spec/cases/any_false.rb").split(',').should == ['false'] * 3 * 3 560 | end 561 | end 562 | 563 | describe ".all?" do 564 | it "returns true if all results are truthy" do 565 | ruby("spec/cases/all_true.rb").split(',').should == ['true'] * 3 * 3 566 | end 567 | 568 | it "returns false if any result is falsy" do 569 | ruby("spec/cases/all_false.rb").split(',').should == ['false'] * 3 * 2 570 | end 571 | end 572 | 573 | describe ".each" do 574 | it "returns original array, works like map" do 575 | ruby("spec/cases/each.rb").should == 'a b c d' 576 | end 577 | 578 | it "passes result to :finish callback :in_processes`" do 579 | monitor = double('monitor', call: nil) 580 | monitor.should_receive(:call).once.with(:first, 0, 123) 581 | monitor.should_receive(:call).once.with(:second, 1, 123) 582 | monitor.should_receive(:call).once.with(:third, 2, 123) 583 | Parallel.each([:first, :second, :third], finish: monitor, in_processes: 3) { 123 } 584 | end 585 | 586 | it "passes result to :finish callback :in_threads`" do 587 | monitor = double('monitor', call: nil) 588 | monitor.should_receive(:call).once.with(:first, 0, 123) 589 | monitor.should_receive(:call).once.with(:second, 1, 123) 590 | monitor.should_receive(:call).once.with(:third, 2, 123) 591 | Parallel.each([:first, :second, :third], finish: monitor, in_threads: 3) { 123 } 592 | end 593 | 594 | it "does not use marshal_dump" do 595 | ruby("spec/cases/no_dump_with_each.rb 2>&1").should == 'no dump for resultno dump for each' 596 | end 597 | 598 | it "does not slow down with lots of GC work in threads" do 599 | Benchmark.realtime { ruby("spec/cases/no_gc_with_each.rb 2>&1") }.should <= 10 600 | end 601 | 602 | it "can modify in-place" do 603 | ruby("spec/cases/each_in_place.rb").should == 'ab' 604 | end 605 | 606 | worker_types.each do |type| 607 | it "works with SQLite in #{type}" do 608 | out = `WORKER_TYPE=#{type} ruby spec/cases/each_with_ar_sqlite.rb 2>&1` 609 | out.gsub!(/.* deprecated; use BigDecimal.*\n/, '') 610 | skip "unsupported" if type == "ractors" 611 | without_ractor_warning(out).should == "Parent: X\nParallel: XXX\nParent: X\n" 612 | end 613 | 614 | it "stops all workers when one fails in #{type}" do 615 | `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception.rb 2>&1`.should =~ /^\d{4} raised$/ 616 | end 617 | 618 | it "stops all workers when one raises Break in #{type}" do 619 | out = `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_break.rb 2>&1` 620 | without_ractor_warning(out).should =~ /^\d{4} Parallel::Break raised - result nil$/ 621 | end 622 | 623 | it "stops all workers when a start hook fails with #{type}" do 624 | out = `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start.rb 2>&1` 625 | without_ractor_warning(out).should =~ /^\d{3} raised$/ 626 | end 627 | 628 | it "does not add new work when a finish hook fails with #{type}" do 629 | out = `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_finish.rb 2>&1` 630 | without_ractor_warning(out).should =~ /^\d{4} raised$/ 631 | end 632 | 633 | it "does not call the finish hook when a worker fails with #{type}" do 634 | out = `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_before_finish.rb 2>&1` 635 | without_ractor_warning(out).should == '3 called' 636 | end 637 | 638 | it "does not call the finish hook when a worker raises Break in #{type}" do 639 | out = `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_break_before_finish.rb 2>&1` 640 | out.should =~ /^\d{3}(finish hook called){3} Parallel::Break raised$/ 641 | end 642 | 643 | it "does not call the finish hook when a start hook fails with #{type}" do 644 | out = `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start_before_finish.rb 2>&1` 645 | if type == "ractors" 646 | # we are calling on the main thread, so everything sleeps 647 | without_ractor_warning(out).should == "start 0\n" 648 | else 649 | out.split("\n").sort.join("\n").should == <<~OUT.rstrip 650 | call 3 651 | finish 2 652 | start 0 653 | start 1 654 | start 2 655 | start 3 656 | OUT 657 | end 658 | end 659 | 660 | it "sets Parallel.worker_number with #{type}" do 661 | skip "unsupported" if type == "ractors" 662 | out = `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_worker_number.rb 2>&1` 663 | out.should =~ /\A[0123]+\z/ 664 | ['0', '1', '2', '3'].each { |number| out.should include number } 665 | end 666 | end 667 | 668 | it "re-raises exceptions in work_direct" do 669 | `METHOD=each WORKER_TYPE=threads WORKER_SIZE=0 ruby spec/cases/with_exception.rb 2>&1` 670 | .should =~ /^1 raised$/ 671 | end 672 | 673 | it "handles Break in work_direct" do 674 | `METHOD=each WORKER_TYPE=threads WORKER_SIZE=0 ruby spec/cases/with_break.rb 2>&1` 675 | .should =~ /^1 Parallel::Break raised - result nil$/ 676 | end 677 | end 678 | 679 | describe ".each_with_index" do 680 | it "yields object and index" do 681 | ["a0b1", "b1a0"].should include ruby("spec/cases/each_with_index.rb 2>&1") 682 | end 683 | end 684 | 685 | describe "progress" do 686 | it "takes the title from :progress" do 687 | ruby("spec/cases/progress.rb 2>&1").sub(/=+/, '==').strip.should == "Doing stuff: |==|" 688 | end 689 | 690 | it "takes true from :progress" do 691 | `TITLE=true ruby spec/cases/progress.rb 2>&1`.sub(/=+/, '==').strip.should == "Progress: |==|" 692 | end 693 | 694 | it "works with :finish" do 695 | ruby("spec/cases/progress_with_finish.rb 2>&1").strip.sub(/=+/, '==').gsub( 696 | /\n+/, 697 | "\n" 698 | ).should == "Doing stuff: |==|\n100" 699 | end 700 | 701 | it "takes the title from :progress[:title] and passes options along" do 702 | ruby("spec/cases/progress_with_options.rb 2>&1").should =~ /Reticulating Splines ;+ \d+ ;+/ 703 | end 704 | end 705 | 706 | ["lambda", "queue"].each do |thing| 707 | describe thing do 708 | let(:result) { "ITEM-1\nITEM-2\nITEM-3\n" } 709 | 710 | worker_types.each do |type| 711 | it "runs in #{type}" do 712 | out = ruby("spec/cases/with_#{thing}.rb #{type} 2>&1") 713 | without_ractor_warning(out).should == result 714 | end 715 | end 716 | 717 | it "refuses to use progress" do 718 | lambda { 719 | Parallel.map(-> {}, progress: "xxx") { raise "Ooops" } # rubocop:disable Lint/UnreachableLoop 720 | }.should raise_error("Progressbar can only be used with array like items") 721 | end 722 | end 723 | end 724 | 725 | it "fails when running with a prefilled queue without stop since there are no threads to fill it" do 726 | error = (RUBY_VERSION >= "2.0.0" ? "No live threads left. Deadlock?" : "deadlock detected (fatal)") 727 | ruby("spec/cases/fatal_queue.rb 2>&1").should include error 728 | end 729 | 730 | describe "GC" do 731 | def normalize(result) 732 | result = result.sub(/\{(.*)\}/, "\\1").split(", ") 733 | result.reject! { |x| x =~ /^(Hash|Array|String)=>(1|-1|-2)$/ } 734 | result.reject! { |x| x =~ /^(Thread::Mutex)=>(1)$/ } if RUBY_VERSION >= "3.3" 735 | result 736 | end 737 | 738 | worker_types.each do |type| 739 | it "does not leak memory in #{type}" do 740 | pending if RUBY_ENGINE == 'jruby' # lots of objects ... GC does not seem to work ... 741 | skip if RUBY_VERSION > "3.4.0" # randomly fails, so can't use pending 742 | options = (RUBY_ENGINE == 'jruby' ? "-X+O" : "") 743 | result = ruby("#{options} spec/cases/profile_memory.rb #{type} 2>&1").strip.split("\n").last 744 | normalize(result).should == [] 745 | end 746 | end 747 | end 748 | end 749 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'parallel' 3 | require 'benchmark' 4 | require 'timeout' 5 | 6 | RSpec.configure do |config| 7 | config.expect_with(:rspec) { |c| c.syntax = :should } 8 | config.mock_with(:rspec) { |c| c.syntax = :should } 9 | config.around { |example| Timeout.timeout(30, &example) } 10 | config.include( 11 | Module.new do 12 | def ruby(cmd) 13 | `#{RbConfig.ruby} #{cmd}` 14 | end 15 | end 16 | ) 17 | end 18 | --------------------------------------------------------------------------------