├── .github ├── dependabot.yml └── workflows │ ├── linting.yml │ └── test.yml ├── .gitignore ├── .rubocop.yml ├── Appraisals ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── bin ├── console └── setup ├── exe ├── cucumber-queue ├── minitest-queue ├── rspec-queue └── testunit-queue ├── gemfiles ├── cucumber1_3.gemfile ├── cucumber2_4.gemfile ├── minitest5.gemfile ├── rspec3.gemfile ├── rspec4.gemfile ├── testunit.gemfile └── turnip.gemfile ├── lib ├── test-queue.rb ├── test_queue.rb └── test_queue │ ├── iterator.rb │ ├── runner.rb │ ├── runner │ ├── cucumber.rb │ ├── example.rb │ ├── minitest.rb │ ├── minitest5.rb │ ├── puppet_lint.rb │ ├── rspec.rb │ ├── rspec_ext.rb │ └── testunit.rb │ ├── stats.rb │ ├── test_framework.rb │ └── version.rb ├── spec └── stats_spec.rb ├── test-queue.gemspec └── test ├── cucumber.bats ├── examples ├── example_minispec.rb ├── example_minitest4.rb ├── example_minitest5.rb ├── example_rspec_helper.rb ├── example_shared_examples_for_spec.rb ├── example_spec.rb ├── example_split_spec.rb ├── example_testunit.rb ├── example_testunit_split.rb ├── example_use_shared_example1_spec.rb ├── example_use_shared_example2_spec.rb └── features │ ├── bad.feature │ ├── example.feature │ ├── example2.feature │ ├── step_definitions │ └── common.rb │ └── turnip_steps │ └── global_steps.rb ├── minitest5.bats ├── rspec3.bats ├── rspec4.bats ├── sleepy_runner.rb ├── testlib.bash ├── testunit.bats └── turnip.bats /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: 'github-actions' 4 | directory: '/' 5 | schedule: 6 | interval: 'weekly' 7 | -------------------------------------------------------------------------------- /.github/workflows/linting.yml: -------------------------------------------------------------------------------- 1 | name: Linting 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | 9 | jobs: 10 | main: 11 | name: RuboCop 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v4 16 | - name: Set up Ruby 2.7 17 | uses: ruby/setup-ruby@v1 18 | with: 19 | ruby-version: 2.7 20 | - name: Build and run RuboCop 21 | run: | 22 | bundle install --jobs 4 --retry 3 23 | bundle exec rubocop 24 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | 9 | jobs: 10 | main: 11 | name: >- 12 | ${{ matrix.ruby }} ${{ matrix.entry.name }} 13 | runs-on: ${{ matrix.os }}-latest 14 | env: 15 | TEST_QUEUE_WORKERS: 2 16 | TEST_QUEUE_VERBOSE: 1 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | os: [ubuntu] 21 | # Lowest and Latest version. 22 | ruby: ['2.7', '3.3'] 23 | entry: 24 | - { name: cucumber1_3, bats: test/cucumber.bats } 25 | - { name: cucumber2_4, bats: test/cucumber.bats } 26 | - { name: minitest5, bats: test/minitest5.bats } 27 | - { name: rspec3, bats: test/rspec3.bats } 28 | - { name: rspec4, bats: test/rspec4.bats } 29 | - { name: testunit, bats: test/testunit.bats } 30 | - { name: turnip, bats: test/turnip.bats } 31 | 32 | steps: 33 | - name: checkout 34 | uses: actions/checkout@v4 35 | - name: set up Ruby 36 | uses: ruby/setup-ruby@v1 37 | with: 38 | ruby-version: ${{ matrix.ruby }} 39 | 40 | - name: install dependencies 41 | run: bundle install --jobs 3 --retry 3 42 | - name: setup for Bats 43 | run: bundle exec rake setup 44 | - name: spec 45 | run: bundle exec rake spec 46 | - name: install dependencies for ${{ matrix.entry.name }} 47 | run: BUNDLE_GEMFILE=gemfiles/${{ matrix.entry.name }}.gemfile bundle install --jobs 3 --retry 3 48 | - name: test 49 | run: BUNDLE_GEMFILE=gemfiles/${{ matrix.entry.name }}.gemfile vendor/bats/bin/bats ${{ matrix.entry.bats }} 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .bundle 2 | .test_queue_stats 3 | .ruby-version 4 | Gemfile.lock 5 | gemfiles/.bundle/ 6 | *.gemfile.lock 7 | pkg 8 | vendor 9 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | require: 2 | - rubocop-performance 3 | 4 | AllCops: 5 | TargetRubyVersion: 2.7 6 | DisabledByDefault: true 7 | NewCops: enable 8 | SuggestExtensions: false 9 | 10 | Bundler: 11 | Enabled: true 12 | 13 | Gemspec: 14 | Enabled: true 15 | 16 | Layout: 17 | Enabled: true 18 | 19 | Layout/LineLength: 20 | Enabled: false 21 | 22 | Lint: 23 | Enabled: true 24 | 25 | Lint/FlipFlop: 26 | Enabled: false 27 | 28 | # This cop is practically style, not lint. And this rule is unexpected. 29 | Lint/AmbiguousOperatorPrecedence: 30 | Enabled: false 31 | 32 | Performance: 33 | Enabled: true 34 | 35 | Performance/Casecmp: 36 | Enabled: false 37 | 38 | Security: 39 | Enabled: true 40 | 41 | # `Marshal.load` is only used to deserialize data that has been serialized internally, 42 | # and not for deserializing data from external sources. 43 | Security/MarshalLoad: 44 | Enabled: false 45 | 46 | Style/Alias: 47 | Enabled: true 48 | 49 | Style/ColonMethodCall: 50 | Enabled: true 51 | 52 | Style/EmptyCaseCondition: 53 | Enabled: true 54 | 55 | Style/FrozenStringLiteralComment: 56 | Enabled: true 57 | 58 | Style/HashSyntax: 59 | Enabled: true 60 | 61 | Style/InfiniteLoop: 62 | Enabled: true 63 | 64 | Style/MethodDefParentheses: 65 | Enabled: true 66 | 67 | Style/NegatedIf: 68 | Enabled: true 69 | 70 | Style/GlobalStdStream: 71 | Enabled: true 72 | 73 | Style/PreferredHashMethods: 74 | Enabled: true 75 | 76 | Style/RedundantBegin: 77 | Enabled: true 78 | 79 | Style/RedundantConstantBase: 80 | Enabled: true 81 | 82 | Style/RedundantParentheses: 83 | Enabled: true 84 | 85 | Style/RedundantRegexpEscape: 86 | Enabled: true 87 | 88 | Style/RedundantReturn: 89 | Enabled: true 90 | 91 | Style/RedundantSelf: 92 | Enabled: true 93 | 94 | Style/SafeNavigation: 95 | Enabled: true 96 | 97 | Style/SelectByRegexp: 98 | Enabled: true 99 | 100 | Style/Semicolon: 101 | Enabled: true 102 | 103 | Style/SignalException: 104 | Enabled: true 105 | 106 | Style/SingleLineMethods: 107 | Enabled: true 108 | 109 | Style/StderrPuts: 110 | Enabled: true 111 | 112 | Style/StringLiterals: 113 | Enabled: true 114 | 115 | Style/StringLiteralsInInterpolation: 116 | Enabled: true 117 | 118 | Style/TrailingCommaInArrayLiteral: 119 | Enabled: true 120 | 121 | Style/VariableInterpolation: 122 | Enabled: true 123 | -------------------------------------------------------------------------------- /Appraisals: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | appraise 'cucumber1-3' do 4 | gem 'cucumber', '~> 1.3.10' 5 | # Pin Rake version to Prevent `NoMethodError: undefined method `last_comment'`. 6 | gem 'rake', '< 11.0' 7 | end 8 | 9 | appraise 'cucumber2-4' do 10 | gem 'cucumber', '~> 2.4.0' 11 | # Pin Rake version to Prevent `NoMethodError: undefined method `last_comment'`. 12 | gem 'rake', '< 11.0' 13 | end 14 | 15 | appraise 'minitest5' do 16 | gem 'rake' 17 | gem 'minitest', '5.10.0' 18 | end 19 | 20 | appraise 'rspec3' do 21 | gem 'rspec', '~> 3.12' 22 | end 23 | 24 | appraise 'rspec4' do 25 | gem 'rspec', github: 'rspec/rspec-metagem', branch: '4-0-dev' 26 | gem 'rspec-core', github: 'rspec/rspec-core', branch: '4-0-dev' 27 | gem 'rspec-expectations', github: 'rspec/rspec-expectations', branch: '4-0-dev' 28 | gem 'rspec-mocks', github: 'rspec/rspec-mocks', branch: '4-0-dev' 29 | gem 'rspec-support', github: 'rspec/rspec-support', branch: '4-0-dev' 30 | end 31 | 32 | appraise 'testunit' do 33 | gem 'test-unit' 34 | end 35 | 36 | appraise 'turnip' do 37 | gem 'turnip', '~> 4.4' 38 | end 39 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | gemspec 6 | 7 | gem 'appraisal' 8 | gem 'rake' 9 | gem 'rspec', '>= 2.13', '< 4.0' 10 | gem 'rubocop', '~> 1.53.0' 11 | gem 'rubocop-performance', '~> 1.17.0' 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Aman Gupta 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # test-queue 2 | 3 | [![Gem Version](https://badge.fury.io/rb/test-queue.svg)](https://rubygems.org/gems/test-queue) 4 | [![CI](https://github.com/tmm1/test-queue/actions/workflows/test.yml/badge.svg)](https://github.com/tmm1/test-queue/actions/workflows/test.yml) 5 | 6 | Yet another parallel test runner, built using a centralized queue to ensure 7 | optimal distribution of tests between workers. 8 | 9 | Specifically optimized for CI environments: build statistics from each run 10 | are stored locally and used to sort the queue at the beginning of the 11 | next run. 12 | 13 | ## Usage 14 | 15 | test-queue bundles `testunit-queue`, `minitest-queue`, and `rspec-queue` binaries which can be used directly: 16 | 17 | ```console 18 | $ minitest-queue $(find test/ -name \*_test.rb) 19 | $ rspec-queue --format progress spec 20 | ``` 21 | 22 | But the underlying `TestQueue::Runner::TestUnit`, `TestQueue::Runner::Minitest`, and `TestQueue::Runner::RSpec` are 23 | built to be subclassed by your application. I recommend checking a new 24 | executable into your project using one of these superclasses. 25 | 26 | ```console 27 | $ vim script/test-queue 28 | $ chmod +x script/test-queue 29 | $ git add script/test-queue 30 | ``` 31 | 32 | Since test-queue uses `fork(2)` to spawn off workers, you must ensure each worker 33 | runs in an isolated environment. Use the `after_fork` hook with a custom 34 | runner to reset any global state. 35 | 36 | ``` ruby 37 | #!/usr/bin/env ruby 38 | 39 | class MyAppTestRunner < TestQueue::Runner::Minitest 40 | def after_fork(num) 41 | # Use separate mysql database (we assume it exists and has the right schema already) 42 | ActiveRecord::Base.configurations.configs_for(env_name: 'test', name: 'primary').database << num.to_s 43 | ActiveRecord::Base.establish_connection(:test) 44 | 45 | # Use separate redis database 46 | $redis.client.db = num 47 | $redis.client.reconnect 48 | end 49 | 50 | def prepare(concurrency) 51 | # Create mysql databases exists with correct schema 52 | concurrency.times do |i| 53 | # ... 54 | end 55 | 56 | # If this is a remote master, tell the central master something about us 57 | @remote_master_message = "Output for remote master 123: http://myhost.com/build/123" 58 | end 59 | 60 | def around_filter(suite) 61 | $stats.timing("test.#{suite}.runtime") do 62 | yield 63 | end 64 | end 65 | end 66 | 67 | MyAppTestRunner.new.execute 68 | ``` 69 | 70 | ## Environment variables 71 | 72 | - `TEST_QUEUE_WORKERS`: Number of workers to use per master (default: all available cores) 73 | - `TEST_QUEUE_VERBOSE`: Show results as they are available (default: `0`) 74 | - `TEST_QUEUE_SOCKET`: Unix socket `path` (or TCP `address:port` pair) used for communication (default: `/tmp/test_queue_XXXXX.sock`) 75 | - `TEST_QUEUE_RELAY`: Relay results back to a central master, specified as TCP `address:port` 76 | - `TEST_QUEUE_STATS`: `path` to cache build stats in-build CI runs (default: `.test_queue_stats`) 77 | - `TEST_QUEUE_FORCE`: Comma separated list of suites to run 78 | - `TEST_QUEUE_RELAY_TIMEOUT`: When using distributed builds, the amount of time a remote master will try to reconnect to start work 79 | - `TEST_QUEUE_RELAY_TOKEN`: When using distributed builds, this must be the same on remote masters and the central master for remote masters to be able to connect. 80 | - `TEST_QUEUE_REMOTE_MASTER_MESSAGE`: When using distributed builds, set this on a remote master and it will appear in that master's connection message on the central master. 81 | - `TEST_QUEUE_SPLIT_GROUPS`: Split tests up by example rather than example group. Faster for tests with short setup time such as selenium. RSpec only. Add the `:no_split` tag to `ExampleGroups` you don't want split. 82 | 83 | ## Design 84 | 85 | test-queue uses a simple master + pre-fork worker model. The master 86 | exposes a Unix domain socket server which workers use to grab tests off 87 | the queue. 88 | 89 | ```console 90 | ─┬─ 21232 minitest-queue master 91 | ├─── 21571 minitest-queue worker [3] - AuthenticationTest 92 | ├─── 21568 minitest-queue worker [2] - ApiTest 93 | ├─── 21565 minitest-queue worker [1] - UsersControllerTest 94 | └─── 21562 minitest-queue worker [0] - UserTest 95 | ``` 96 | 97 | test-queue also has a distributed mode, where additional masters can share 98 | the workload and relay results back to a central master. 99 | 100 | ## Distributed mode 101 | 102 | To use distributed mode, the central master must listen on a TCP port. Additional masters can be booted 103 | in relay mode to connect to the central master. Remote masters must provide a `TEST_QUEUE_RELAY_TOKEN` 104 | to match the central master's. 105 | 106 | ```console 107 | $ TEST_QUEUE_RELAY_TOKEN=123 TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec minitest-queue ./test/example_test.rb 108 | $ TEST_QUEUE_RELAY_TOKEN=123 TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec minitest-queue ./test/example_test.rb 109 | $ TEST_QUEUE_RELAY_TOKEN=123 ./test-multi.sh 110 | ``` 111 | 112 | See the [Parameterized Trigger Plugin](https://wiki.jenkins-ci.org/display/JENKINS/Parameterized+Trigger+Plugin) 113 | for a simple way to do this with Jenkins. 114 | 115 | ## See also 116 | 117 | - https://github.com/Shopify/rails_parallel 118 | - https://github.com/grosser/parallel_tests 119 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'bundler/gem_tasks' 4 | require 'rspec/core/rake_task' 5 | require 'rubocop/rake_task' 6 | 7 | RSpec::Core::RakeTask.new(:spec) 8 | RuboCop::RakeTask.new 9 | 10 | task default: [:setup, :rubocop, :spec, :feature] 11 | 12 | task :setup do 13 | sh 'bin/setup' unless File.exist?("#{Dir.pwd}/vendor/bats/bin/bats") 14 | end 15 | 16 | task :feature do 17 | sh 'TEST_QUEUE_WORKERS=2 TEST_QUEUE_VERBOSE=1 vendor/bats/bin/bats test' 18 | end 19 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require 'bundler/setup' 5 | require 'irb' 6 | require 'test_queue' 7 | 8 | ARGV.clear 9 | 10 | IRB.start(__FILE__) 11 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | cd "$(dirname "$0")/.." 6 | ROOT=$(pwd) 7 | 8 | rm -rf tmp 9 | mkdir tmp 10 | mkdir -p vendor/bats 11 | curl --silent --location --show-error https://github.com/sstephenson/bats/archive/v0.4.0.tar.gz | tar -xz -C tmp 12 | tmp/bats-0.4.0/install.sh "$ROOT/vendor/bats" 13 | rm -rf tmp 14 | 15 | bundle install --jobs 3 --retry 3 16 | -------------------------------------------------------------------------------- /exe/cucumber-queue: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | $LOAD_PATH.unshift File.expand_path('../lib', __dir__) 5 | 6 | require 'test_queue' 7 | require 'test_queue/runner/cucumber' 8 | 9 | TestQueue::Runner::Cucumber.new.execute 10 | -------------------------------------------------------------------------------- /exe/minitest-queue: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | $LOAD_PATH.unshift File.expand_path('../lib', __dir__) 5 | 6 | require 'test_queue' 7 | require 'test_queue/runner/minitest' 8 | 9 | TestQueue::Runner::Minitest.new.execute 10 | -------------------------------------------------------------------------------- /exe/rspec-queue: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | $LOAD_PATH.unshift File.expand_path('../lib', __dir__) 5 | 6 | require 'test_queue' 7 | require 'test_queue/runner/rspec' 8 | 9 | TestQueue::Runner::RSpec.new.execute 10 | -------------------------------------------------------------------------------- /exe/testunit-queue: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | $LOAD_PATH.unshift File.expand_path('../lib', __dir__) 5 | 6 | require 'test_queue' 7 | require 'test_queue/runner/testunit' 8 | 9 | TestQueue::Runner::TestUnit.new.execute 10 | -------------------------------------------------------------------------------- /gemfiles/cucumber1_3.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # This file was generated by Appraisal 4 | 5 | source 'https://rubygems.org' 6 | 7 | gem 'cucumber', '~> 1.3.10' 8 | gem 'rake', '< 11.0' 9 | gem 'rspec', '>= 2.13', '< 4.0' 10 | 11 | gemspec path: '../' 12 | -------------------------------------------------------------------------------- /gemfiles/cucumber2_4.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # This file was generated by Appraisal 4 | 5 | source 'https://rubygems.org' 6 | 7 | gem 'cucumber', '~> 2.4.0' 8 | gem 'rake', '< 11.0' 9 | gem 'rspec', '>= 2.13', '< 4.0' 10 | 11 | gemspec path: '../' 12 | -------------------------------------------------------------------------------- /gemfiles/minitest5.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # This file was generated by Appraisal 4 | 5 | source 'https://rubygems.org' 6 | 7 | gem 'minitest', '5.10.0' 8 | 9 | gemspec path: '../' 10 | -------------------------------------------------------------------------------- /gemfiles/rspec3.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # This file was generated by Appraisal 4 | 5 | source 'https://rubygems.org' 6 | 7 | gem 'rspec', '~> 3.12' 8 | 9 | gemspec path: '../' 10 | -------------------------------------------------------------------------------- /gemfiles/rspec4.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # This file was generated by Appraisal 4 | 5 | source 'https://rubygems.org' 6 | 7 | gem 'rspec', github: 'rspec/rspec-metagem', branch: '4-0-dev' 8 | gem 'rspec-core', github: 'rspec/rspec-core', branch: '4-0-dev' 9 | gem 'rspec-expectations', github: 'rspec/rspec-expectations', branch: '4-0-dev' 10 | gem 'rspec-mocks', github: 'rspec/rspec-mocks', branch: '4-0-dev' 11 | gem 'rspec-support', github: 'rspec/rspec-support', branch: '4-0-dev' 12 | 13 | gemspec path: '../' 14 | -------------------------------------------------------------------------------- /gemfiles/testunit.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # This file was generated by Appraisal 4 | 5 | source 'https://rubygems.org' 6 | 7 | gem 'test-unit' 8 | 9 | gemspec path: '../' 10 | -------------------------------------------------------------------------------- /gemfiles/turnip.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # This file was generated by Appraisal 4 | 5 | source 'https://rubygems.org' 6 | 7 | gem 'turnip', '~> 4.4' 8 | 9 | gemspec path: '../' 10 | -------------------------------------------------------------------------------- /lib/test-queue.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'test_queue' 4 | -------------------------------------------------------------------------------- /lib/test_queue.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'test_queue/iterator' 4 | require_relative 'test_queue/runner' 5 | -------------------------------------------------------------------------------- /lib/test_queue/iterator.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module TestQueue 4 | class Iterator 5 | include Enumerable 6 | 7 | attr_reader :sock 8 | 9 | def initialize(test_framework, sock, filter = nil, run_token:, early_failure_limit: nil) 10 | @test_framework = test_framework 11 | @done = false 12 | @suite_stats = [] 13 | @sock = sock 14 | @filter = filter 15 | if @sock =~ /\A(.+):(\d+)\z/ 16 | @tcp_address = $1 17 | @tcp_port = $2.to_i 18 | end 19 | @failures = 0 20 | @early_failure_limit = early_failure_limit 21 | @run_token = run_token 22 | end 23 | 24 | def each 25 | raise "already used this iterator. previous caller: #{@done}" if @done 26 | 27 | procline = $0 28 | 29 | loop do 30 | # If we've hit too many failures in one worker, assume the entire 31 | # test suite is broken, and notify master so the run 32 | # can be immediately halted. 33 | if @early_failure_limit && @failures >= @early_failure_limit 34 | connect_to_master('KABOOM') 35 | break 36 | else 37 | client = connect_to_master("POP #{Socket.gethostname} #{Process.pid}") 38 | end 39 | break if client.nil? 40 | 41 | # This false positive will be resolved by https://github.com/rubocop/rubocop/pull/11830. 42 | _r, _w, e = IO.select([client], nil, [client], nil) 43 | break unless e.empty? 44 | 45 | if (data = client.read(65536)) 46 | client.close 47 | item = Marshal.load(data) 48 | break if item.nil? || item.empty? 49 | 50 | if item == 'WAIT' 51 | $0 = "#{procline} - Waiting for work" 52 | sleep 0.1 53 | next 54 | end 55 | suite_name, path = item 56 | suite = load_suite(suite_name, path) 57 | 58 | # Maybe we were told to load a suite that doesn't exist anymore. 59 | next unless suite 60 | 61 | $0 = "#{procline} - #{suite.respond_to?(:description) ? suite.description : suite}" 62 | start = Time.now 63 | if @filter 64 | @filter.call(suite) { yield suite } 65 | else 66 | yield suite 67 | end 68 | @suite_stats << TestQueue::Stats::Suite.new(suite_name, path, Time.now - start, Time.now) 69 | @failures += suite.failure_count if suite.respond_to? :failure_count 70 | else 71 | break 72 | end 73 | end 74 | rescue Errno::ENOENT, Errno::ECONNRESET, Errno::ECONNREFUSED 75 | # noop 76 | ensure 77 | $0 = procline 78 | @done = caller(1..1).first 79 | File.open("/tmp/test_queue_worker_#{$$}_suites", 'wb') do |f| 80 | Marshal.dump(@suite_stats, f) 81 | end 82 | end 83 | 84 | def connect_to_master(cmd) 85 | sock = 86 | if @tcp_address 87 | TCPSocket.new(@tcp_address, @tcp_port) 88 | else 89 | UNIXSocket.new(@sock) 90 | end 91 | sock.puts("TOKEN=#{@run_token}") 92 | sock.puts(cmd) 93 | sock 94 | rescue Errno::EPIPE 95 | nil 96 | end 97 | 98 | def empty? 99 | false 100 | end 101 | 102 | def load_suite(suite_name, path) 103 | @loaded_suites ||= {} 104 | suite = @loaded_suites[suite_name] 105 | return suite if suite 106 | 107 | @test_framework.suites_from_file(path).each do |name, suite_from_file| 108 | @loaded_suites[name] = suite_from_file 109 | end 110 | @loaded_suites[suite_name] 111 | end 112 | end 113 | end 114 | -------------------------------------------------------------------------------- /lib/test_queue/runner.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'set' 4 | require 'socket' 5 | require 'fileutils' 6 | require 'securerandom' 7 | require_relative 'stats' 8 | require_relative 'test_framework' 9 | 10 | module TestQueue 11 | class Worker 12 | attr_accessor :pid, :status, :output, :num, :host 13 | attr_accessor :start_time, :end_time 14 | attr_accessor :summary, :failure_output 15 | 16 | # Array of TestQueue::Stats::Suite recording all the suites this worker ran. 17 | attr_reader :suites 18 | 19 | def initialize(pid, num) 20 | @pid = pid 21 | @num = num 22 | @start_time = Time.now 23 | @output = '' 24 | @suites = [] 25 | end 26 | 27 | def lines 28 | @output.split("\n") 29 | end 30 | end 31 | 32 | class Runner 33 | attr_accessor :concurrency, :exit_when_done 34 | attr_reader :stats 35 | 36 | TOKEN_REGEX = /\ATOKEN=(\w+)/ 37 | 38 | def initialize(test_framework, concurrency = nil, socket = nil, relay = nil) 39 | @test_framework = test_framework 40 | @stats = Stats.new(stats_file) 41 | 42 | @early_failure_limit = nil 43 | if ENV['TEST_QUEUE_EARLY_FAILURE_LIMIT'] 44 | begin 45 | @early_failure_limit = Integer(ENV['TEST_QUEUE_EARLY_FAILURE_LIMIT']) 46 | rescue ArgumentError 47 | raise ArgumentError, 'TEST_QUEUE_EARLY_FAILURE_LIMIT could not be parsed as an integer' 48 | end 49 | end 50 | 51 | @procline = $0 52 | 53 | @allowlist = if (forced = ENV['TEST_QUEUE_FORCE']) 54 | forced.split(/\s*,\s*/) 55 | else 56 | [] 57 | end 58 | @allowlist.freeze 59 | 60 | all_files = @test_framework.all_suite_files.to_set 61 | @queue = @stats.all_suites 62 | .select { |suite| all_files.include?(suite.path) } 63 | .sort_by { |suite| -suite.duration } 64 | .map { |suite| [suite.name, suite.path] } 65 | 66 | if @allowlist.any? 67 | @queue.select! { |suite_name, _path| @allowlist.include?(suite_name) } 68 | @queue.sort_by! { |suite_name, _path| @allowlist.index(suite_name) } 69 | end 70 | 71 | @awaited_suites = Set.new(@allowlist) 72 | @original_queue = Set.new(@queue).freeze 73 | 74 | @workers = {} 75 | @completed = [] 76 | 77 | @concurrency = concurrency || ENV['TEST_QUEUE_WORKERS']&.to_i || 78 | if File.exist?('/proc/cpuinfo') 79 | File.read('/proc/cpuinfo').split("\n").grep(/processor/).size 80 | elsif RUBY_PLATFORM.include?('darwin') 81 | `/usr/sbin/sysctl -n hw.activecpu`.to_i 82 | else 83 | 2 84 | end 85 | unless @concurrency > 0 86 | raise ArgumentError, "Worker count (#{@concurrency}) must be greater than 0" 87 | end 88 | 89 | @relay_connection_timeout = ENV['TEST_QUEUE_RELAY_TIMEOUT']&.to_i || 30 90 | @run_token = ENV['TEST_QUEUE_RELAY_TOKEN'] || SecureRandom.hex(8) 91 | @socket = socket || ENV['TEST_QUEUE_SOCKET'] || "/tmp/test_queue_#{$$}_#{object_id}.sock" 92 | @relay = relay || ENV['TEST_QUEUE_RELAY'] 93 | @remote_master_message = ENV['TEST_QUEUE_REMOTE_MASTER_MESSAGE'] if ENV.key?('TEST_QUEUE_REMOTE_MASTER_MESSAGE') 94 | 95 | if @relay == @socket 96 | warn '*** Detected TEST_QUEUE_RELAY == TEST_QUEUE_SOCKET. Disabling relay mode.' 97 | @relay = nil 98 | elsif @relay 99 | @queue = [] 100 | end 101 | 102 | @discovered_suites = Set.new 103 | @assignments = {} 104 | 105 | @exit_when_done = true 106 | 107 | @aborting = false 108 | end 109 | 110 | # Run the tests. 111 | # 112 | # If exit_when_done is true, exit! will be called before this method 113 | # completes. If exit_when_done is false, this method will return an Integer 114 | # number of failures. 115 | def execute 116 | $stdout.sync = $stderr.sync = true 117 | @start_time = Time.now 118 | 119 | execute_internal 120 | exitstatus = summarize_internal 121 | 122 | if exit_when_done 123 | exit! exitstatus 124 | else 125 | exitstatus 126 | end 127 | end 128 | 129 | def summarize_internal 130 | puts 131 | puts "==> Summary (#{@completed.size} workers in %.4fs)" % (Time.now - @start_time) 132 | puts 133 | 134 | estatus = 0 135 | misrun_suites = [] 136 | unassigned_suites = [] 137 | @failures = '' 138 | @completed.each do |worker| 139 | estatus += (worker.status.exitstatus || 1) 140 | @stats.record_suites(worker.suites) 141 | worker.suites.each do |suite| 142 | assignment = @assignments.delete([suite.name, suite.path]) 143 | host = worker.host || Socket.gethostname 144 | if assignment.nil? 145 | unassigned_suites << [suite.name, suite.path] 146 | elsif assignment != [host, worker.pid] 147 | misrun_suites << [suite.name, suite.path] + assignment + [host, worker.pid] 148 | end 149 | @discovered_suites.delete([suite.name, suite.path]) 150 | end 151 | 152 | summarize_worker(worker) 153 | 154 | @failures += worker.failure_output if worker.failure_output 155 | 156 | puts ' [%2d] %60s %4d suites in %.4fs (%s %s)' % [ 157 | worker.num, 158 | worker.summary, 159 | worker.suites.size, 160 | worker.end_time - worker.start_time, 161 | worker.status.to_s, 162 | worker.host && " on #{worker.host.split('.').first}" 163 | ] 164 | end 165 | 166 | unless @failures.empty? 167 | puts 168 | puts '==> Failures' 169 | puts 170 | puts @failures 171 | end 172 | 173 | unless relay? 174 | unless @discovered_suites.empty? 175 | estatus += 1 176 | puts 177 | puts 'The following suites were discovered but were not run:' 178 | puts 179 | 180 | @discovered_suites.sort.each do |suite_name, path| 181 | puts "#{suite_name} - #{path}" 182 | end 183 | end 184 | unless unassigned_suites.empty? 185 | estatus += 1 186 | puts 187 | puts 'The following suites were not discovered but were run anyway:' 188 | puts 189 | unassigned_suites.sort.each do |suite_name, path| 190 | puts "#{suite_name} - #{path}" 191 | end 192 | end 193 | unless misrun_suites.empty? 194 | estatus += 1 195 | puts 196 | puts 'The following suites were run on the wrong workers:' 197 | puts 198 | misrun_suites.each do |suite_name, path, target_host, target_pid, actual_host, actual_pid| 199 | puts "#{suite_name} - #{path}: #{actual_host} (#{actual_pid}) - assigned to #{target_host} (#{target_pid})" 200 | end 201 | end 202 | end 203 | 204 | puts 205 | 206 | @stats.save 207 | 208 | summarize 209 | 210 | estatus = @completed.inject(0) { |s, worker| s + (worker.status.exitstatus || 1) } 211 | [estatus, 255].min 212 | end 213 | 214 | def summarize 215 | end 216 | 217 | def stats_file 218 | ENV['TEST_QUEUE_STATS'] || '.test_queue_stats' 219 | end 220 | 221 | def execute_internal 222 | start_master 223 | prepare(@concurrency) 224 | @prepared_time = Time.now 225 | start_relay if relay? 226 | discover_suites 227 | spawn_workers 228 | distribute_queue 229 | ensure 230 | stop_master 231 | 232 | kill_subprocesses 233 | end 234 | 235 | def start_master 236 | unless relay? 237 | if @socket =~ /\A(?:(.+):)?(\d+)\z/ 238 | address = $1 || '0.0.0.0' 239 | port = $2.to_i 240 | @socket = "#{$1}:#{$2}" 241 | @server = TCPServer.new(address, port) 242 | else 243 | FileUtils.rm_f(@socket) 244 | @server = UNIXServer.new(@socket) 245 | end 246 | end 247 | 248 | desc = "test-queue master (#{relay? ? "relaying to #{@relay}" : @socket})" 249 | puts "Starting #{desc}" 250 | $0 = "#{desc} - #{@procline}" 251 | end 252 | 253 | def start_relay 254 | return unless relay? 255 | 256 | sock = connect_to_relay 257 | message = @remote_master_message ? " #{@remote_master_message}" : '' 258 | message = message.gsub(/(\r|\n)/, '') # Our "protocol" is newline-separated 259 | sock.puts("TOKEN=#{@run_token}") 260 | sock.puts("REMOTE MASTER #{@concurrency} #{Socket.gethostname} #{message}") 261 | response = sock.gets.strip 262 | unless response == 'OK' 263 | warn "*** Got non-OK response from master: #{response}" 264 | sock.close 265 | exit! 1 266 | end 267 | sock.close 268 | rescue Errno::ECONNREFUSED 269 | warn "*** Unable to connect to relay #{@relay}. Aborting..." 270 | exit! 1 271 | end 272 | 273 | def stop_master 274 | return if relay? 275 | 276 | FileUtils.rm_f(@socket) if @socket && @server.is_a?(UNIXServer) 277 | @server.close rescue nil if @server 278 | @socket = @server = nil 279 | end 280 | 281 | def spawn_workers 282 | @concurrency.times do |i| 283 | num = i + 1 284 | 285 | pid = Process.fork do 286 | @server&.close 287 | 288 | iterator = Iterator.new(@test_framework, relay? ? @relay : @socket, method(:around_filter), early_failure_limit: @early_failure_limit, run_token: @run_token) 289 | after_fork_internal(num, iterator) 290 | ret = run_worker(iterator) || 0 291 | cleanup_worker 292 | Kernel.exit! ret 293 | end 294 | 295 | @workers[pid] = Worker.new(pid, num) 296 | end 297 | end 298 | 299 | def discover_suites 300 | # Remote masters don't discover suites; the central master does and 301 | # distributes them to remote masters. 302 | return if relay? 303 | 304 | # No need to discover suites if all allowlisted suites are already 305 | # queued. 306 | return if @allowlist.any? && @awaited_suites.empty? 307 | 308 | @discovering_suites_pid = fork do 309 | terminate = false 310 | Signal.trap('INT') { terminate = true } 311 | 312 | $0 = 'test-queue suite discovery process' 313 | 314 | @test_framework.all_suite_files.each do |path| 315 | @test_framework.suites_from_file(path).each do |suite_name, _suite| 316 | Kernel.exit!(0) if terminate 317 | 318 | @server.connect_address.connect do |sock| 319 | sock.puts("TOKEN=#{@run_token}") 320 | data = Marshal.dump([suite_name, path]) 321 | sock.puts("NEW SUITE #{data.bytesize}") 322 | sock.write(data) 323 | end 324 | end 325 | end 326 | 327 | Kernel.exit! 0 328 | end 329 | end 330 | 331 | def awaiting_suites? 332 | # We're waiting to find all the allowlisted suites so we can run them in the correct order. 333 | # Or we don't have any suites yet, but we're working on it. 334 | if @awaited_suites.any? || @queue.empty? && !!@discovering_suites_pid 335 | true 336 | else 337 | # It's fine to run any queued suites now. 338 | false 339 | end 340 | end 341 | 342 | def enqueue_discovered_suite(suite_name, path) 343 | if @allowlist.any? && !@allowlist.include?(suite_name) 344 | return 345 | end 346 | 347 | @discovered_suites << [suite_name, path] 348 | 349 | if @original_queue.include?([suite_name, path]) 350 | # This suite was already added to the queue some other way. 351 | @awaited_suites.delete(suite_name) 352 | return 353 | end 354 | 355 | # We don't know how long new suites will take to run, so we put them at 356 | # the front of the queue. It's better to run a fast suite early than to 357 | # run a slow suite late. 358 | @queue.unshift [suite_name, path] 359 | 360 | if @awaited_suites.delete?(suite_name) && @awaited_suites.empty? 361 | # We've found all the allowlisted suites. Sort the queue to match the 362 | # allowlist. 363 | @queue.sort_by! { |queued_suite_name, _path| @allowlist.index(queued_suite_name) } 364 | 365 | kill_suite_discovery_process('INT') 366 | end 367 | end 368 | 369 | def after_fork_internal(num, iterator) 370 | srand 371 | 372 | output = File.open("/tmp/test_queue_worker_#{$$}_output", 'w') 373 | 374 | $stdout.reopen(output) 375 | $stderr.reopen($stdout) 376 | $stdout.sync = $stderr.sync = true 377 | 378 | $0 = "test-queue worker [#{num}]" 379 | puts 380 | puts "==> Starting #{$0} (#{Process.pid} on #{Socket.gethostname}) - iterating over #{iterator.sock}" 381 | puts 382 | 383 | after_fork(num) 384 | end 385 | 386 | # Run in the master before the fork. Used to create 387 | # concurrency copies of any databases required by the 388 | # test workers. 389 | def prepare(concurrency) 390 | end 391 | 392 | def around_filter(_suite) 393 | yield 394 | end 395 | 396 | # Prepare a worker for executing jobs after a fork. 397 | def after_fork(num) 398 | end 399 | 400 | # Entry point for internal runner implementations. The iterator will yield 401 | # jobs from the shared queue on the master. 402 | # 403 | # Returns an Integer number of failures. 404 | def run_worker(iterator) 405 | iterator.each do |item| 406 | puts " #{item.inspect}" 407 | end 408 | 409 | 0 # exit status 410 | end 411 | 412 | def cleanup_worker 413 | end 414 | 415 | def summarize_worker(worker) 416 | worker.summary = '' 417 | worker.failure_output = '' 418 | end 419 | 420 | def reap_workers(blocking = true) 421 | @workers.delete_if do |_, worker| 422 | if Process.waitpid(worker.pid, blocking ? 0 : Process::WNOHANG).nil? 423 | next false 424 | end 425 | 426 | worker.status = $? 427 | worker.end_time = Time.now 428 | 429 | collect_worker_data(worker) 430 | relay_to_master(worker) if relay? 431 | worker_completed(worker) 432 | 433 | true 434 | end 435 | end 436 | 437 | def collect_worker_data(worker) 438 | if File.exist?(file = "/tmp/test_queue_worker_#{worker.pid}_output") 439 | worker.output = File.binread(file) 440 | FileUtils.rm(file) 441 | end 442 | 443 | if File.exist?(file = "/tmp/test_queue_worker_#{worker.pid}_suites") 444 | worker.suites.replace(Marshal.load(File.binread(file))) 445 | FileUtils.rm(file) 446 | end 447 | end 448 | 449 | def worker_completed(worker) 450 | return if @aborting 451 | 452 | @completed << worker 453 | puts worker.output if ENV['TEST_QUEUE_VERBOSE'] || worker.status.exitstatus != 0 454 | end 455 | 456 | def distribute_queue 457 | return if relay? 458 | 459 | remote_workers = 0 460 | 461 | until !awaiting_suites? && @queue.empty? && remote_workers == 0 462 | queue_status(@start_time, @queue.size, @workers.size, remote_workers) 463 | 464 | if (status = reap_suite_discovery_process(false)) 465 | abort('Discovering suites failed.') unless status.success? 466 | abort("Failed to discover #{@awaited_suites.sort.join(', ')} specified in TEST_QUEUE_FORCE") if @awaited_suites.any? 467 | end 468 | 469 | if @server.wait_readable(0.1).nil? 470 | reap_workers(false) # check for worker deaths 471 | else 472 | sock = @server.accept 473 | token = sock.gets.strip 474 | cmd = sock.gets.strip 475 | 476 | token = token[TOKEN_REGEX, 1] 477 | # If we have a remote master from a different test run, respond with "WRONG RUN", and it will consider the test run done. 478 | if token != @run_token 479 | message = token.nil? ? 'Worker sent no token to master' : "Worker from run #{token} connected to master" 480 | warn "*** #{message} for run #{@run_token}; ignoring." 481 | sock.write("WRONG RUN\n") 482 | next 483 | end 484 | 485 | case cmd 486 | when /\APOP (\S+) (\d+)/ 487 | hostname = $1 488 | pid = Integer($2) 489 | if awaiting_suites? 490 | sock.write(Marshal.dump('WAIT')) 491 | elsif (obj = @queue.shift) 492 | data = Marshal.dump(obj) 493 | sock.write(data) 494 | @assignments[obj] = [hostname, pid] 495 | end 496 | when /\AREMOTE MASTER (\d+) ([\w.-]+)(?: (.+))?/ 497 | num = $1.to_i 498 | remote_master = $2 499 | remote_master_message = $3 500 | 501 | sock.write("OK\n") 502 | remote_workers += num 503 | 504 | message = "*** #{num} workers connected from #{remote_master} after #{Time.now - @start_time}s" 505 | message += " #{remote_master_message}" if remote_master_message 506 | warn message 507 | when /\AWORKER (\d+)/ 508 | data = sock.read($1.to_i) 509 | worker = Marshal.load(data) 510 | worker_completed(worker) 511 | remote_workers -= 1 512 | when /\ANEW SUITE (\d+)/ 513 | data = sock.read($1.to_i) 514 | suite_name, path = Marshal.load(data) 515 | enqueue_discovered_suite(suite_name, path) 516 | when /\AKABOOM/ 517 | # worker reporting an abnormal number of test failures; 518 | # stop everything immediately and report the results. 519 | break 520 | else 521 | warn("Ignoring unrecognized command: \"#{cmd}\"") 522 | end 523 | sock.close 524 | end 525 | end 526 | ensure 527 | stop_master 528 | reap_workers 529 | end 530 | 531 | def relay? 532 | !!@relay 533 | end 534 | 535 | def connect_to_relay 536 | sock = nil 537 | start = Time.now 538 | puts "Attempting to connect for #{@relay_connection_timeout}s..." 539 | while sock.nil? 540 | begin 541 | sock = TCPSocket.new(*@relay.split(':')) 542 | rescue Errno::ECONNREFUSED => e 543 | raise e if Time.now - start > @relay_connection_timeout 544 | 545 | puts 'Master not yet available, sleeping...' 546 | sleep 0.5 547 | end 548 | end 549 | sock 550 | end 551 | 552 | def relay_to_master(worker) 553 | worker.host = Socket.gethostname 554 | data = Marshal.dump(worker) 555 | 556 | sock = connect_to_relay 557 | sock.puts("TOKEN=#{@run_token}") 558 | sock.puts("WORKER #{data.bytesize}") 559 | sock.write(data) 560 | ensure 561 | sock&.close 562 | end 563 | 564 | def kill_subprocesses 565 | kill_workers 566 | kill_suite_discovery_process 567 | end 568 | 569 | def kill_workers 570 | @workers.each do |pid, _worker| 571 | Process.kill 'KILL', pid 572 | end 573 | 574 | reap_workers 575 | end 576 | 577 | def kill_suite_discovery_process(signal = 'KILL') 578 | return unless @discovering_suites_pid 579 | 580 | Process.kill signal, @discovering_suites_pid 581 | reap_suite_discovery_process 582 | end 583 | 584 | def reap_suite_discovery_process(blocking = true) 585 | return unless @discovering_suites_pid 586 | 587 | _, status = Process.waitpid2(@discovering_suites_pid, blocking ? 0 : Process::WNOHANG) 588 | return unless status 589 | 590 | @discovering_suites_pid = nil 591 | status 592 | end 593 | 594 | # Stop the test run immediately. 595 | # 596 | # message - String message to print to the console when exiting. 597 | # 598 | # Doesn't return. 599 | def abort(message) 600 | @aborting = true 601 | kill_subprocesses 602 | Kernel.abort("Aborting: #{message}") 603 | end 604 | 605 | # Subclasses can override to monitor the status of the queue. 606 | # 607 | # For example, you may want to record metrics about how quickly remote 608 | # workers connect, or abort the build if not enough connect. 609 | # 610 | # This method is called very frequently during the test run, so don't do 611 | # anything expensive/blocking. 612 | # 613 | # This method is not called on remote masters when using remote workers, 614 | # only on the central master. 615 | # 616 | # start_time - Time when the test run began 617 | # queue_size - Integer number of suites left in the queue 618 | # local_worker_count - Integer number of active local workers 619 | # remote_worker_count - Integer number of active remote workers 620 | # 621 | # Returns nothing. 622 | def queue_status(start_time, queue_size, local_worker_count, remote_worker_count) 623 | end 624 | end 625 | end 626 | -------------------------------------------------------------------------------- /lib/test_queue/runner/cucumber.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'cucumber' 4 | require 'cucumber/rspec/disable_option_parser' 5 | require 'cucumber/cli/main' 6 | 7 | module Cucumber 8 | module Ast 9 | class Features 10 | attr_accessor :features 11 | end 12 | 13 | class Feature 14 | def to_s 15 | title 16 | end 17 | end 18 | end 19 | 20 | class Runtime 21 | if defined?(::Cucumber::Runtime::FeaturesLoader) 22 | # Without this module, Runtime#features would load all features specified 23 | # on the command line. We want to avoid that and load only the features 24 | # each worker needs ourselves, so we override the default behavior to let 25 | # us put our iterator in place without loading any features directly. 26 | module InjectableFeatures 27 | def features 28 | return @features if defined?(@features) 29 | 30 | super 31 | end 32 | 33 | def features=(iterator) 34 | @features = ::Cucumber::Ast::Features.new 35 | @features.features = iterator 36 | end 37 | end 38 | 39 | prepend InjectableFeatures 40 | else 41 | attr_writer :features 42 | end 43 | end 44 | end 45 | 46 | module TestQueue 47 | class Runner 48 | class Cucumber < Runner 49 | def initialize 50 | super(TestFramework::Cucumber.new) 51 | end 52 | 53 | def run_worker(iterator) 54 | runtime = @test_framework.runtime 55 | runtime.features = iterator 56 | 57 | @test_framework.cli.execute!(runtime) 58 | 59 | if runtime.respond_to?(:summary_report, true) 60 | runtime.send(:summary_report).test_cases.total_failed 61 | else 62 | runtime.results.scenarios(:failed).size 63 | end 64 | end 65 | 66 | def summarize_worker(worker) 67 | output = worker.output.gsub(/\e\[\d+./, '') 68 | worker.summary = output.split("\n").grep(/^\d+ (scenarios?|steps?)/).first 69 | worker.failure_output = output.scan(/^Failing Scenarios:\n(.*)\n\d+ scenarios?/m).join("\n") 70 | end 71 | end 72 | end 73 | 74 | class TestFramework 75 | class Cucumber < TestFramework 76 | class FakeKernel 77 | def exit(n) 78 | if $! 79 | # Let Cucumber exit for raised exceptions. 80 | Kernel.exit(n) 81 | end 82 | # Don't let Cucumber exit to indicate test failures. We want to 83 | # return the number of failures from #run_worker instead. 84 | end 85 | end 86 | 87 | def cli 88 | @cli ||= ::Cucumber::Cli::Main.new(ARGV.dup, $stdin, $stdout, $stderr, FakeKernel.new) 89 | end 90 | 91 | def runtime 92 | @runtime ||= ::Cucumber::Runtime.new(cli.configuration) 93 | end 94 | 95 | def all_suite_files 96 | if runtime.respond_to?(:feature_files, true) 97 | runtime.send(:feature_files) 98 | else 99 | cli.configuration.feature_files 100 | end 101 | end 102 | 103 | def suites_from_file(path) 104 | if defined?(::Cucumber::Core::Gherkin::Document) 105 | source = ::Cucumber::Runtime::NormalisedEncodingFile.read(path) 106 | doc = ::Cucumber::Core::Gherkin::Document.new(path, source) 107 | [[File.basename(doc.uri), doc]] 108 | else 109 | loader = 110 | ::Cucumber::Runtime::FeaturesLoader.new([path], 111 | cli.configuration.filters, 112 | cli.configuration.tag_expression) 113 | loader.features.map { |feature| [feature.title, feature] } 114 | end 115 | end 116 | end 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /lib/test_queue/runner/example.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../../test_queue' 4 | require_relative '../runner' 5 | 6 | module TestQueue 7 | class Runner 8 | class Example < Runner 9 | def initialize(args) 10 | super(TestFramework::Example.new(args)) 11 | end 12 | 13 | def spawn_workers 14 | puts "Spawning #{@concurrency} workers" 15 | super 16 | end 17 | 18 | def after_fork(num) 19 | puts " -- worker #{num} booted as pid #{$$}" 20 | super 21 | end 22 | 23 | def run_worker(iterator) 24 | iterator.inject(0) do |sum, item| 25 | puts " #{item.inspect}" 26 | sum + item.to_i 27 | end 28 | end 29 | 30 | def summarize_worker(worker) 31 | worker.summary = worker.output.scan(/^\s*(\d+)/).join(', ') 32 | worker.failure_output = '' 33 | end 34 | end 35 | end 36 | 37 | class TestFramework 38 | class Example < TestFramework 39 | def initialize(args) 40 | super() 41 | 42 | @args = args.map(&:to_s) 43 | end 44 | 45 | def all_suite_files 46 | @args 47 | end 48 | 49 | def suites_from_file(_path) 50 | @args.map { |i| [i, i] } 51 | end 52 | end 53 | end 54 | end 55 | 56 | if __FILE__ == $0 57 | TestQueue::Runner::Example.new(Array(1..10)).execute 58 | end 59 | 60 | __END__ 61 | 62 | Spawning 4 workers 63 | -- worker 0 booted as pid 40406 64 | -- worker 1 booted as pid 40407 65 | -- worker 2 booted as pid 40408 66 | -- worker 3 booted as pid 40409 67 | 68 | ==> Starting ruby test-queue worker [1] (40407) 69 | 70 | 2 71 | 5 72 | 8 73 | 74 | ==> Starting ruby test-queue worker [3] (40409) 75 | 76 | 77 | ==> Starting ruby test-queue worker [2] (40408) 78 | 79 | 3 80 | 6 81 | 9 82 | 83 | ==> Starting ruby test-queue worker [0] (40406) 84 | 85 | 1 86 | 4 87 | 7 88 | 10 89 | 90 | ==> Summary 91 | 92 | [1] 2, 5, 8 in 0.0024s (pid 40407 exit 15) 93 | [3] in 0.0036s (pid 40409 exit 0) 94 | [2] 3, 6, 9 in 0.0038s (pid 40408 exit 18) 95 | [0] 1, 4, 7, 10 in 0.0044s (pid 40406 exit 22) 96 | -------------------------------------------------------------------------------- /lib/test_queue/runner/minitest.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'minitest' 4 | 5 | raise 'requires Minitest version 5' unless Minitest::VERSION.to_i == 5 6 | 7 | require_relative '../runner/minitest5' 8 | 9 | module TestQueue 10 | class Runner 11 | class Minitest < Runner 12 | def summarize_worker(worker) 13 | worker.summary = worker.lines.grep(/, \d+ errors?, /).first 14 | failures = worker.lines.select { |line| 15 | line if (line =~ /^Finished/) ... (line =~ /, \d+ errors?, /) 16 | }[1..-2] 17 | worker.failure_output = failures.join("\n") if failures 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/test_queue/runner/minitest5.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../runner' 4 | 5 | module Minitest 6 | def self.__run(reporter, options) 7 | suites = Runnable.runnables 8 | suites.map { |suite| suite.run reporter, options } 9 | end 10 | 11 | class Runnable 12 | def failure_count 13 | failures.length 14 | end 15 | end 16 | 17 | class Test 18 | def self.runnables=(runnables) 19 | @@runnables = runnables 20 | end 21 | 22 | # Synchronize all tests, even serial ones. 23 | # 24 | # Minitest runs serial tests before parallel ones to ensure the 25 | # unsynchronized serial tests don't overlap the parallel tests. But since 26 | # the test-queue master hands out tests without actually loading their 27 | # code, there's no way to know which are parallel and which are serial. 28 | # Synchronizing serial tests does add some overhead, but hopefully this is 29 | # outweighed by the speed benefits of using test-queue. 30 | def _synchronize 31 | Test.io_lock.synchronize { yield } 32 | end 33 | end 34 | 35 | class ProgressReporter 36 | # Override original method to make test-queue specific output 37 | def record(result) 38 | io.print ' ' 39 | io.print result.class 40 | io.print ': ' 41 | io.print result.result_code 42 | io.puts(' <%.3f>' % result.time) 43 | end 44 | end 45 | 46 | begin 47 | require 'minitest/minitest_reporter_plugin' 48 | 49 | class << self 50 | private 51 | 52 | def total_count(_options) 53 | 0 54 | end 55 | end 56 | rescue LoadError 57 | # noop 58 | end 59 | end 60 | 61 | module TestQueue 62 | class Runner 63 | class Minitest < Runner 64 | def initialize 65 | @options = ::Minitest.process_args ARGV 66 | 67 | if ::Minitest.respond_to?(:seed) 68 | ::Minitest.seed = @options[:seed] 69 | srand ::Minitest.seed 70 | end 71 | 72 | if ::Minitest::Test.runnables.any? { |r| r.runnable_methods.any? } 73 | raise 'Do not `require` test files. Pass them via ARGV instead and they will be required as needed.' 74 | end 75 | 76 | super(TestFramework::Minitest.new) 77 | end 78 | 79 | def start_master 80 | puts "Run options: #{@options[:args]}\n\n" 81 | 82 | super 83 | end 84 | 85 | def run_worker(iterator) 86 | ::Minitest::Test.runnables = iterator 87 | ::Minitest.run ? 0 : 1 88 | end 89 | end 90 | MiniTest = Minitest # For compatibility with test-queue 0.7.0 and earlier. 91 | end 92 | 93 | class TestFramework 94 | class Minitest < TestFramework 95 | def all_suite_files 96 | ARGV 97 | end 98 | 99 | def suites_from_file(path) 100 | ::Minitest::Test.reset 101 | require File.absolute_path(path) 102 | ::Minitest::Test.runnables 103 | .reject { |s| s.runnable_methods.empty? } 104 | .map { |s| [s.name, s] } 105 | end 106 | end 107 | MiniTest = Minitest # For compatibility with test-queue 0.7.0 and earlier. 108 | end 109 | end 110 | -------------------------------------------------------------------------------- /lib/test_queue/runner/puppet_lint.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../../test_queue' 4 | require 'puppet-lint' 5 | 6 | module TestQueue 7 | class Runner 8 | class PuppetLint < Runner 9 | def run_worker(iterator) 10 | errors = 0 11 | linter = PuppetLint.new 12 | iterator.each do |file| 13 | puts "Evaluating #{file}" 14 | linter.file = file 15 | linter.run 16 | errors += 1 if linter.errors? 17 | end 18 | errors 19 | end 20 | 21 | def summarize_worker(worker) 22 | lines = worker.lines 23 | 24 | files = lines.grep(/^Evaluating/) 25 | errors = lines.grep(/^ERROR/) 26 | warnings = lines.grep(/^WARNING/) 27 | 28 | worker.summary = "#{files.size} files, #{warnings.size} warnings, #{errors.size} errors" 29 | worker.failure_output = errors.join("\n") 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/test_queue/runner/rspec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rspec/core' 4 | 5 | raise 'requires RSpec version 3 or 4' unless [3, 4].include?(RSpec::Core::Version::STRING.to_i) 6 | 7 | require_relative 'rspec_ext' 8 | require_relative '../runner' 9 | 10 | module TestQueue 11 | class Runner 12 | class RSpec < Runner 13 | def initialize 14 | super(TestFramework::RSpec.new) 15 | 16 | @rspec = ::RSpec::Core::QueueRunner.new 17 | end 18 | 19 | def start_master 20 | seed_notification = ::RSpec::Core::Notifications::SeedNotification.new(@rspec.configuration.seed, seed_used?) 21 | puts "#{seed_notification.fully_formatted}\n" 22 | 23 | super 24 | end 25 | 26 | def run_worker(iterator) 27 | @rspec.run_each(iterator).to_i 28 | end 29 | 30 | def summarize_worker(worker) 31 | worker.summary = worker.lines.grep(/ examples?, /).first 32 | worker.failure_output = worker.output[/^Failures:\n\n(.*)\n^Finished/m, 1] 33 | end 34 | 35 | private 36 | 37 | def seed_used? 38 | @rspec.configuration.seed && @rspec.configuration.seed_used? 39 | end 40 | end 41 | end 42 | 43 | class TestFramework 44 | class RSpec < TestFramework 45 | begin 46 | require 'turnip/rspec' 47 | 48 | include Turnip::RSpec::Loader 49 | rescue LoadError 50 | # noop 51 | end 52 | 53 | def all_suite_files 54 | options = ::RSpec::Core::ConfigurationOptions.new(ARGV) 55 | options.parse_options if options.respond_to?(:parse_options) 56 | options.configure(::RSpec.configuration) 57 | 58 | if ::RSpec.configuration.instance_variable_defined?(:@files_or_directories_to_run) && 59 | ::RSpec.configuration.instance_variable_get(:@files_or_directories_to_run).empty? 60 | ::RSpec.configuration.instance_variable_set(:@files_or_directories_to_run, [::RSpec.configuration.default_path]) 61 | end 62 | 63 | ::RSpec.configuration.files_to_run.uniq 64 | end 65 | 66 | def suites_from_file(path) 67 | ::RSpec.world.example_groups.clear 68 | load path 69 | split_groups(::RSpec.world.example_groups).map { |example_or_group| 70 | name = if example_or_group.respond_to?(:id) 71 | example_or_group.id 72 | elsif example_or_group.respond_to?(:full_description) 73 | example_or_group.full_description 74 | elsif example_or_group.metadata.key?(:full_description) 75 | example_or_group.metadata[:full_description] 76 | else 77 | example_or_group.metadata[:example_group][:full_description] 78 | end 79 | [name, example_or_group] 80 | } 81 | end 82 | 83 | private 84 | 85 | def split_groups(groups) 86 | return groups unless split_groups? 87 | 88 | groups_to_split, groups_to_keep = [], [] 89 | groups.each do |group| 90 | (group.metadata[:no_split] ? groups_to_keep : groups_to_split) << group 91 | end 92 | queue = groups_to_split.flat_map(&:descendant_filtered_examples) 93 | queue.concat groups_to_keep 94 | queue 95 | end 96 | 97 | def split_groups? 98 | return @split_groups if defined?(@split_groups) 99 | 100 | @split_groups = ENV['TEST_QUEUE_SPLIT_GROUPS'] && ENV['TEST_QUEUE_SPLIT_GROUPS'].strip.downcase == 'true' 101 | end 102 | end 103 | end 104 | end 105 | -------------------------------------------------------------------------------- /lib/test_queue/runner/rspec_ext.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class ::RSpec::Core::ExampleGroup 4 | def self.failure_count 5 | examples.map { |e| e.execution_result.status == 'failed' }.length 6 | end 7 | end 8 | 9 | module RSpec::Core 10 | # RSpec 3.2 introduced: 11 | unless Configuration.method_defined?(:with_suite_hooks) 12 | class Configuration 13 | def with_suite_hooks 14 | hook_context = SuiteHookContext.new 15 | hooks.run(:before, :suite, hook_context) 16 | yield 17 | ensure 18 | hooks.run(:after, :suite, hook_context) 19 | end 20 | end 21 | end 22 | 23 | class QueueRunner < Runner 24 | def initialize 25 | options = ConfigurationOptions.new(ARGV) 26 | super(options) 27 | end 28 | 29 | def run_specs(iterator) 30 | @configuration.reporter.report(0) do |reporter| 31 | @configuration.with_suite_hooks do 32 | iterator.map { |g| 33 | start = Time.now 34 | if g.is_a? ::RSpec::Core::Example 35 | print " #{g.full_description}: " 36 | example = g 37 | g = example.example_group 38 | ::RSpec.world.filtered_examples.clear 39 | ::RSpec.world.filtered_examples[g] = [example] 40 | else 41 | print " #{g.description}: " 42 | end 43 | ret = g.run(reporter) 44 | diff = Time.now - start 45 | puts(' <%.3f>' % diff) 46 | 47 | ret 48 | }.all? ? 0 : @configuration.failure_exit_code 49 | end 50 | end 51 | end 52 | alias run_each run_specs 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/test_queue/runner/testunit.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../runner' 4 | 5 | gem 'test-unit' 6 | require 'test/unit' 7 | require 'test/unit/collector/load' 8 | require 'test/unit/ui/console/testrunner' 9 | 10 | module TestQueue 11 | class Runner 12 | class TestUnit < Runner 13 | class TestSuite < ::Test::Unit::TestSuite 14 | def initialize(name, iterator) 15 | super(name) 16 | @tests = IteratorWrapper.new(iterator) 17 | end 18 | 19 | def run(*) 20 | @started = true 21 | super 22 | end 23 | 24 | def size 25 | return 0 unless @started 26 | 27 | super 28 | end 29 | end 30 | 31 | class IteratorWrapper 32 | def initialize(iterator) 33 | @generator = Fiber.new do 34 | iterator.each do |test| 35 | Fiber.yield(test) 36 | end 37 | end 38 | end 39 | 40 | def shift 41 | @generator.resume 42 | rescue FiberError 43 | nil 44 | end 45 | 46 | def each 47 | while (test = shift) 48 | yield(test) 49 | end 50 | end 51 | end 52 | 53 | def initialize 54 | super(TestFramework::TestUnit.new) 55 | end 56 | 57 | def run_worker(iterator) 58 | @suite = TestSuite.new('specified by test-queue master', iterator) 59 | res = Test::Unit::UI::Console::TestRunner.new(@suite).start 60 | res.run_count - res.pass_count 61 | end 62 | 63 | def summarize_worker(worker) 64 | worker.summary = worker.output.split("\n").grep(/^\d+ tests?/).first 65 | worker.failure_output = worker.output.scan(/^Failure:[^\n]*\n(.*?)\n=======================*/m).join("\n") 66 | end 67 | end 68 | end 69 | 70 | class TestFramework 71 | class TestUnit < TestFramework 72 | def all_suite_files 73 | ARGV 74 | end 75 | 76 | def suites_from_file(path) 77 | test_suite = Test::Unit::Collector::Load.new.collect(path) 78 | return [] unless test_suite 79 | return test_suite.tests.map { [_1.name, _1] } unless split_groups? 80 | 81 | split_groups(test_suite) 82 | end 83 | 84 | def split_groups? 85 | return @split_groups if defined?(@split_groups) 86 | 87 | @split_groups = ENV['TEST_QUEUE_SPLIT_GROUPS'] && ENV['TEST_QUEUE_SPLIT_GROUPS'].strip.downcase == 'true' 88 | end 89 | 90 | def split_groups(test_suite, groups = []) 91 | unless splittable?(test_suite) 92 | groups << [test_suite.name, test_suite] 93 | return groups 94 | end 95 | 96 | test_suite.tests.each do |suite| 97 | if suite.is_a?(Test::Unit::TestSuite) 98 | split_groups(suite, groups) 99 | else 100 | groups << [suite.name, suite] 101 | end 102 | end 103 | groups 104 | end 105 | 106 | def splittable?(test_suite) 107 | test_suite.tests.none? do |test| 108 | test.is_a?(Test::Unit::TestCase) && test[:no_split] 109 | end 110 | end 111 | end 112 | end 113 | end 114 | -------------------------------------------------------------------------------- /lib/test_queue/stats.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module TestQueue 4 | class Stats 5 | class Suite 6 | attr_reader :name, :path, :duration, :last_seen_at 7 | 8 | def initialize(name, path, duration, last_seen_at) 9 | @name = name 10 | @path = path 11 | @duration = duration 12 | @last_seen_at = last_seen_at 13 | 14 | freeze 15 | end 16 | 17 | def ==(other) 18 | other && 19 | name == other.name && 20 | path == other.path && 21 | duration == other.duration && 22 | last_seen_at == other.last_seen_at 23 | end 24 | alias eql? == 25 | 26 | def to_h 27 | { name: name, path: path, duration: duration, last_seen_at: last_seen_at.to_i } 28 | end 29 | 30 | def self.from_hash(hash) 31 | new(hash.fetch(:name), 32 | hash.fetch(:path), 33 | hash.fetch(:duration), 34 | Time.at(hash.fetch(:last_seen_at))) 35 | end 36 | end 37 | 38 | def initialize(path) 39 | @path = path 40 | @suites = {} 41 | load 42 | end 43 | 44 | def all_suites 45 | @suites.values 46 | end 47 | 48 | def suite(name) 49 | @suites[name] 50 | end 51 | 52 | def record_suites(suites) 53 | suites.each do |suite| 54 | @suites[suite.name] = suite 55 | end 56 | end 57 | 58 | def save 59 | prune 60 | 61 | File.open(@path, 'wb') do |f| 62 | Marshal.dump(to_h, f) 63 | end 64 | end 65 | 66 | private 67 | 68 | CURRENT_VERSION = 2 69 | 70 | def to_h 71 | suites = @suites.each_value.map(&:to_h) 72 | 73 | { version: CURRENT_VERSION, suites: suites } 74 | end 75 | 76 | def load 77 | data = begin 78 | File.open(@path, 'rb') { |f| Marshal.load(f) } 79 | rescue Errno::ENOENT, EOFError, TypeError, ArgumentError 80 | # noop 81 | end 82 | return unless data.is_a?(Hash) && data[:version] == CURRENT_VERSION 83 | 84 | data[:suites].each do |suite_hash| 85 | suite = Suite.from_hash(suite_hash) 86 | @suites[suite.name] = suite 87 | end 88 | end 89 | 90 | EIGHT_DAYS_S = 8 * 24 * 60 * 60 91 | 92 | def prune 93 | earliest = Time.now - EIGHT_DAYS_S 94 | @suites.delete_if do |_name, suite| 95 | suite.last_seen_at < earliest 96 | end 97 | end 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /lib/test_queue/test_framework.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module TestQueue 4 | # This class provides an abstraction over the various test frameworks we 5 | # support. The framework-specific subclasses are defined in the various 6 | # test_queue/runner/* files. 7 | class TestFramework 8 | # Return all file paths to load test suites from. 9 | # 10 | # An example implementation might just return files passed on the command 11 | # line, or defer to the underlying test framework to determine which files 12 | # to load. 13 | # 14 | # Returns an Enumerable of String file paths. 15 | def all_suite_files 16 | raise NotImplementedError 17 | end 18 | 19 | # Load all suites from the specified file path. 20 | # 21 | # path - String file path to load suites from 22 | # 23 | # Returns an Enumerable of tuples containing: 24 | # suite_name - String that uniquely identifies this suite 25 | # suite - Framework-specific object that can be used to actually 26 | # run the suite 27 | def suites_from_file(path) 28 | raise NotImplementedError 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/test_queue/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module TestQueue 4 | VERSION = '0.11.1' 5 | end 6 | -------------------------------------------------------------------------------- /spec/stats_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'fileutils' 4 | require 'tempfile' 5 | 6 | $LOAD_PATH.unshift File.expand_path('../lib', __dir__) 7 | 8 | require 'test_queue/stats' 9 | 10 | RSpec.describe TestQueue::Stats do 11 | before do 12 | Tempfile.open('test_queue_stats') do |f| 13 | @path = f.path 14 | f.close! 15 | end 16 | end 17 | 18 | after do 19 | FileUtils.rm_f(@path) 20 | end 21 | 22 | describe '#initialize' do 23 | it 'ignores empty stats files' do 24 | File.write(@path, '') 25 | stats = TestQueue::Stats.new(@path) 26 | expect(stats.all_suites).to be_empty 27 | end 28 | 29 | it 'ignores invalid data in the stats files' do 30 | File.write(@path, 'this is not marshal data') 31 | stats = TestQueue::Stats.new(@path) 32 | expect(stats.all_suites).to be_empty 33 | end 34 | 35 | it 'ignores badly-typed data in the stats file' do 36 | File.write(@path, Marshal.dump(['heyyy'])) 37 | stats = TestQueue::Stats.new(@path) 38 | expect(stats.all_suites).to be_empty 39 | end 40 | 41 | it 'ignores stats files with a wrong version number' do 42 | File.write(@path, Marshal.dump({ version: 1e8, suites: 'boom' })) 43 | stats = TestQueue::Stats.new(@path) 44 | expect(stats.all_suites).to be_empty 45 | end 46 | end 47 | 48 | it 'can save and load data' do 49 | stats = TestQueue::Stats.new(@path) 50 | time = truncated_now 51 | suites = [ 52 | TestQueue::Stats::Suite.new('Suite1', 'foo.rb', 0.3, time), 53 | TestQueue::Stats::Suite.new('Suite2', 'bar.rb', 0.5, time + 5) 54 | ] 55 | stats.record_suites(suites) 56 | stats.save 57 | 58 | stats = TestQueue::Stats.new(@path) 59 | expect(stats.all_suites.sort_by(&:name)).to eq(suites) 60 | end 61 | 62 | it 'prunes suites not seen in the last 8 days' do 63 | stats = TestQueue::Stats.new(@path) 64 | time = truncated_now 65 | suites = [ 66 | TestQueue::Stats::Suite.new('Suite1', 'foo.rb', 0.3, time), 67 | TestQueue::Stats::Suite.new('Suite2', 'bar.rb', 0.5, time - (8 * 24 * 60 * 60) - 2), 68 | TestQueue::Stats::Suite.new('Suite3', 'baz.rb', 0.6, time - (7 * 24 * 60 * 60)) 69 | ] 70 | stats.record_suites(suites) 71 | stats.save 72 | 73 | stats = TestQueue::Stats.new(@path) 74 | expect(stats.all_suites.map(&:name).sort).to eq(%w[Suite1 Suite3]) 75 | end 76 | 77 | # Returns Time.now rounded down to the nearest second. 78 | def truncated_now 79 | Time.at(Time.now.to_i) 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /test-queue.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'lib/test_queue/version' 4 | 5 | Gem::Specification.new do |s| 6 | s.name = 'test-queue' 7 | s.version = TestQueue::VERSION 8 | s.required_ruby_version = '>= 2.7.0' 9 | s.summary = 'parallel test runner' 10 | s.description = 'minitest/rspec parallel test runner for CI environments' 11 | 12 | s.homepage = 'https://github.com/tmm1/test-queue' 13 | 14 | s.authors = ['Aman Gupta'] 15 | s.email = 'ruby@tmm1.net' 16 | s.license = 'MIT' 17 | 18 | s.bindir = 'exe' 19 | s.executables << 'rspec-queue' 20 | s.executables << 'minitest-queue' 21 | s.executables << 'testunit-queue' 22 | s.executables << 'cucumber-queue' 23 | 24 | s.files = Dir['LICENSE', 'README.md', 'lib/**/*'] 25 | s.metadata['rubygems_mfa_required'] = 'true' 26 | end 27 | -------------------------------------------------------------------------------- /test/cucumber.bats: -------------------------------------------------------------------------------- 1 | load "testlib" 2 | 3 | SCRATCH=tmp/cucumber-tests 4 | 5 | setup() { 6 | require_gem "cucumber" ">= 1.0" 7 | rm -rf $SCRATCH 8 | mkdir -p $SCRATCH 9 | } 10 | 11 | teardown() { 12 | rm -rf $SCRATCH 13 | } 14 | 15 | @test "cucumber-queue succeeds when all features pass" { 16 | run bundle exec cucumber-queue test/examples/features --require test/examples/features/step_definitions 17 | assert_status 0 18 | assert_output_contains "Starting test-queue master" 19 | } 20 | 21 | @test "cucumber-queue fails when a feature fails" { 22 | export FAIL=1 23 | run bundle exec cucumber-queue test/examples/features --require test/examples/features/step_definitions 24 | assert_status 2 25 | assert_output_contains "Starting test-queue master" 26 | assert_output_contains "cucumber test/examples/features/bad.feature:2 # Scenario: failure" 27 | assert_output_contains "cucumber test/examples/features/example2.feature:26 # Scenario: failure" 28 | } 29 | 30 | @test "cucumber-queue fails when given a missing feature" { 31 | run bundle exec cucumber-queue test/examples/does_not_exist.feature --require test/examples/features/step_definitions 32 | assert_status 1 33 | assert_output_contains "Aborting: Discovering suites failed." 34 | } 35 | 36 | @test "cucumber-queue fails when given a malformed feature" { 37 | [ -f README.md ] 38 | run bundle exec cucumber-queue README.md --require test/examples/features/step_definitions 39 | 40 | # Cucumber 1 and 2 fail in different ways. 41 | refute_status 0 42 | assert_output_matches 'Aborting: Discovering suites failed\.|README\.md: Parser errors:' 43 | } 44 | 45 | @test "cucumber-queue handles test file being deleted" { 46 | cp test/examples/features/*.feature $SCRATCH 47 | 48 | run bundle exec cucumber-queue $SCRATCH --require test/examples/features/step_definitions 49 | assert_status 0 50 | assert_output_matches "Feature: Foobar$" 51 | 52 | rm $SCRATCH/example.feature 53 | 54 | run bundle exec cucumber-queue $SCRATCH --require test/examples/features/step_definitions 55 | assert_status 0 56 | refute_output_matches "Feature: Foobar$" 57 | } 58 | -------------------------------------------------------------------------------- /test/examples/example_minispec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'minitest/spec' 4 | 5 | class Meme 6 | def i_can_has_cheezburger? 7 | 'OHAI!' 8 | end 9 | 10 | def will_it_blend? 11 | 'YES!' 12 | end 13 | end 14 | 15 | describe Meme do 16 | before do 17 | @meme = Meme.new 18 | end 19 | 20 | describe 'when asked about cheeseburgers' do 21 | it 'must respond positively' do 22 | sleep 0.1 23 | @meme.i_can_has_cheezburger?.must_equal 'OHAI!' 24 | end 25 | end 26 | 27 | describe 'when asked about blending possibilities' do 28 | it "won't say no" do 29 | sleep 0.1 30 | @meme.will_it_blend?.wont_match(/^no/i) 31 | end 32 | 33 | if ENV['FAIL'] 34 | it 'fails' do 35 | @meme.will_it_blend?.must_equal 'NO!' 36 | end 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /test/examples/example_minitest4.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'minitest/unit' 4 | 5 | class MinitestEqual < MiniTest::Unit::TestCase 6 | def test_equal 7 | assert_equal 1, 1 8 | end 9 | end 10 | 11 | 30.times do |i| 12 | Object.const_set("MinitestSleep#{i}", Class.new(Minitest::Unit::TestCase) do 13 | define_method(:test_sleep) do 14 | start = Time.now 15 | sleep(0.25) 16 | assert_in_delta Time.now - start, 0.25, 0.02 17 | end 18 | end) 19 | end 20 | 21 | if ENV['FAIL'] 22 | class MinitestFailure < MiniTest::Unit::TestCase 23 | def test_fail 24 | assert_equal 0, 1 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /test/examples/example_minitest5.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'minitest/autorun' 4 | 5 | class MinitestEqual < Minitest::Test 6 | def test_equal 7 | assert_equal 1, 1 8 | end 9 | end 10 | 11 | 30.times do |i| 12 | Object.const_set("MinitestSleep#{i}", Class.new(Minitest::Test) do 13 | define_method(:test_sleep) do 14 | start = Time.now 15 | sleep(0.25) 16 | assert_in_delta Time.now - start, 0.25, 0.02 17 | end 18 | end) 19 | end 20 | 21 | if ENV['FAIL'] 22 | class MinitestFailure < Minitest::Test 23 | def test_fail 24 | assert_equal 0, 1 25 | end 26 | end 27 | end 28 | 29 | if ENV['KILL'] 30 | class MinitestKilledFailure < Minitest::Test 31 | def test_kill 32 | Process.kill(9, $$) 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /test/examples/example_rspec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'example_shared_examples_for_spec' 4 | -------------------------------------------------------------------------------- /test/examples/example_shared_examples_for_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.shared_examples_for 'Shared Example' do 4 | it { is_expected.to eq 5 } 5 | end 6 | -------------------------------------------------------------------------------- /test/examples/example_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rspec' 4 | 5 | RSpec.describe 'RSpecEqual' do 6 | it 'checks equality' do 7 | expect(1).to eq 1 8 | end 9 | end 10 | 11 | 30.times do |i| 12 | RSpec.describe "RSpecSleep(#{i})" do 13 | it 'sleeps' do 14 | start = Time.now 15 | sleep(0.25) 16 | expect(Time.now - start).to be_within(0.02).of(0.25) 17 | end 18 | end 19 | end 20 | 21 | if ENV['FAIL'] 22 | RSpec.describe 'RSpecFailure' do 23 | it 'fails' do 24 | expect(:foo).to eq :bar 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /test/examples/example_split_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rspec' 4 | 5 | RSpec.describe 'SplittableGroup', no_split: !!ENV['NOSPLIT'] do 6 | 2.times do |i| 7 | it "runs test #{i}" do 8 | # Sleep longer in CI to make the distribution of examples across workers 9 | # more deterministic. 10 | if ENV['CI'] 11 | sleep(5) 12 | else 13 | sleep(1) 14 | end 15 | 16 | expect(1).to eq 1 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /test/examples/example_testunit.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'test/unit' 4 | 5 | class TestUnitEqual < Test::Unit::TestCase 6 | def test_equal 7 | assert_equal 1, 1 8 | end 9 | end 10 | 11 | class Short < Test::Unit::TestCase 12 | def test_work_with_short_class_name 13 | assert true 14 | end 15 | end 16 | 17 | 30.times do |i| 18 | Object.const_set("TestUnitSleep#{i}", Class.new(Test::Unit::TestCase) do 19 | define_method(:test_sleep) do 20 | start = Time.now 21 | sleep(0.25) 22 | assert_in_delta Time.now - start, 0.25, 0.02 23 | end 24 | end) 25 | end 26 | 27 | if ENV['FAIL'] 28 | class TestUnitFailure < Test::Unit::TestCase 29 | def test_fail 30 | assert_equal 0, 1 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /test/examples/example_testunit_split.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'test/unit' 4 | 5 | class SplittableTestCase < Test::Unit::TestCase 6 | def wait 7 | # Sleep longer in CI to make the distribution of examples across workers 8 | # more deterministic. 9 | if ENV['CI'] 10 | sleep(5) 11 | else 12 | sleep(1) 13 | end 14 | end 15 | 16 | sub_test_case 'splittable sub_test_case 1' do 17 | test 'test 1' do 18 | wait 19 | assert true 20 | end 21 | 22 | sub_test_case 'splittable sub_test_case 2' do 23 | attribute :no_split, !!ENV['NOSPLIT'], keep: true 24 | 25 | test 'test 3' do 26 | wait 27 | assert true 28 | end 29 | 30 | test 'test 4' do 31 | wait 32 | assert true 33 | end 34 | 35 | test 'test 5' do 36 | wait 37 | assert true 38 | end 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /test/examples/example_use_shared_example1_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rspec' 4 | require_relative 'example_rspec_helper' 5 | 6 | RSpec.describe 'Use SharedExamplesFor test_1' do 7 | subject { 5 } 8 | it_behaves_like 'Shared Example' 9 | end 10 | -------------------------------------------------------------------------------- /test/examples/example_use_shared_example2_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rspec' 4 | require_relative 'example_rspec_helper' 5 | 6 | RSpec.describe 'Use SharedExamplesFor test_2' do 7 | subject { 5 } 8 | it_behaves_like 'Shared Example' 9 | end 10 | -------------------------------------------------------------------------------- /test/examples/features/bad.feature: -------------------------------------------------------------------------------- 1 | Feature: bad 2 | Scenario: failure 3 | Given a 4 | When bad 5 | Then c 6 | -------------------------------------------------------------------------------- /test/examples/features/example.feature: -------------------------------------------------------------------------------- 1 | Feature: Foobar 2 | Scenario: Bar 3 | Given a 4 | When b 5 | Then c 6 | Scenario: Baz 7 | Given a 8 | When b 9 | Then c 10 | Scenario: Bam 11 | Given a 12 | When b 13 | Then c 14 | Scenario: Car 15 | Given a 16 | When b 17 | Then c 18 | Scenario: Caz 19 | Given a 20 | When b 21 | Then c 22 | Scenario: Cam 23 | Given a 24 | When b 25 | Then c 26 | -------------------------------------------------------------------------------- /test/examples/features/example2.feature: -------------------------------------------------------------------------------- 1 | Feature: Foobar2 2 | Scenario: Bar 3 | Given a 4 | When b 5 | Then c 6 | Scenario: Baz 7 | Given a 8 | When b 9 | Then c 10 | Scenario: Bam 11 | Given a 12 | When b 13 | Then c 14 | Scenario: Car 15 | Given a 16 | When b 17 | Then c 18 | Scenario: Caz 19 | Given a 20 | When b 21 | Then c 22 | Scenario: Cam 23 | Given a 24 | When b 25 | Then c 26 | Scenario: failure 27 | Given a 28 | When bad 29 | Then c 30 | -------------------------------------------------------------------------------- /test/examples/features/step_definitions/common.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Given(/^a$/) do 4 | sleep 0.10 5 | end 6 | 7 | When(/^b$/) do 8 | sleep 0.25 9 | end 10 | 11 | When(/^bad$/) do 12 | if ENV['FAIL'] 13 | 1.should == 0 14 | else 15 | 1.should == 1 16 | end 17 | end 18 | 19 | Then(/^c$/) do 20 | 1.should == 1 21 | end 22 | -------------------------------------------------------------------------------- /test/examples/features/turnip_steps/global_steps.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | step %(a) do 4 | 1.should == 1 5 | end 6 | 7 | step %(b) do 8 | 1.should == 1 9 | end 10 | 11 | step %(bad) do 12 | if ENV['FAIL'] 13 | 1.should == 0 14 | else 15 | 1.should == 1 16 | end 17 | end 18 | 19 | step %(c) do 20 | 1.should == 1 21 | end 22 | -------------------------------------------------------------------------------- /test/minitest5.bats: -------------------------------------------------------------------------------- 1 | load "testlib" 2 | 3 | SCRATCH=tmp/minitest5-tests 4 | 5 | setup() { 6 | require_gem "minitest" "~> 5.0" 7 | rm -rf $SCRATCH 8 | mkdir -p $SCRATCH 9 | } 10 | 11 | teardown() { 12 | rm -rf $SCRATCH 13 | } 14 | 15 | @test "minitest-queue (minitest5) succeeds when all tests pass" { 16 | run bundle exec minitest-queue ./test/examples/*_minitest5.rb 17 | assert_status 0 18 | assert_output_contains "Starting test-queue master" 19 | } 20 | 21 | @test "minitest-queue (minitest5) succeeds when all tests pass with the --seed option" { 22 | run bundle exec minitest-queue --seed 1234 ./test/examples/*_minitest5.rb 23 | assert_status 0 24 | assert_output_contains "Run options: --seed 1234" 25 | assert_output_contains "Starting test-queue master" 26 | } 27 | 28 | @test "minitest-queue (minitest5) succeeds when all tests pass with the SEED env variable" { 29 | export SEED=1234 30 | run bundle exec minitest-queue ./test/examples/*_minitest5.rb 31 | assert_status 0 32 | assert_output_contains "Run options: --seed 1234" 33 | assert_output_contains "Starting test-queue master" 34 | } 35 | 36 | @test "minitest-queue (minitest5) fails when a test fails" { 37 | export FAIL=1 38 | run bundle exec minitest-queue ./test/examples/*_minitest5.rb 39 | assert_status 1 40 | assert_output_contains "Starting test-queue master" 41 | assert_output_contains "1) Failure:" 42 | assert_output_contains "MinitestFailure#test_fail" 43 | } 44 | 45 | @test "TEST_QUEUE_FORCE allowlists certain tests" { 46 | export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="MinitestSleep11,MinitestSleep8" 47 | run bundle exec minitest-queue ./test/examples/*_minitest5.rb 48 | assert_status 0 49 | assert_output_contains "Starting test-queue master" 50 | assert_output_contains "MinitestSleep11" 51 | assert_output_contains "MinitestSleep8" 52 | refute_output_contains "MinitestSleep9" 53 | } 54 | 55 | assert_test_queue_force_ordering() { 56 | run bundle exec minitest-queue "$@" 57 | assert_status 0 58 | assert_output_contains "Starting test-queue master" 59 | 60 | # Turn the list of suites that were run into a comma-separated list. Input 61 | # looks like: 62 | # SuiteName: . <0.001> 63 | actual_tests=$(echo "$output" | \ 64 | egrep '^ .*: \.+ <' | \ 65 | sed -E -e 's/^ (.*): \.+.*/\1/' | \ 66 | tr '\n' ',' | \ 67 | sed -e 's/,$//') 68 | assert_equal "$TEST_QUEUE_FORCE" "$actual_tests" 69 | } 70 | 71 | @test "TEST_QUEUE_FORCE ensures test ordering" { 72 | export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="Meme::when asked about cheeseburgers,MinitestEqual" 73 | 74 | # Without stats file 75 | rm -f .test_queue_stats 76 | assert_test_queue_force_ordering ./test/examples/example_minitest5.rb ./test/examples/example_minispec.rb 77 | rm -f .test_queue_stats 78 | assert_test_queue_force_ordering ./test/examples/example_minispec.rb ./test/examples/example_minitest5.rb 79 | 80 | # With stats file 81 | assert_test_queue_force_ordering ./test/examples/example_minitest5.rb ./test/examples/example_minispec.rb 82 | assert_test_queue_force_ordering ./test/examples/example_minispec.rb ./test/examples/example_minitest5.rb 83 | } 84 | 85 | @test "minitest-queue fails if TEST_QUEUE_FORCE specifies nonexistent tests" { 86 | export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="MinitestSleep11,DoesNotExist" 87 | run bundle exec minitest-queue ./test/examples/*_minitest5.rb 88 | assert_status 1 89 | assert_output_contains "Failed to discover DoesNotExist specified in TEST_QUEUE_FORCE" 90 | } 91 | 92 | @test "multi-master central master succeeds when all tests pass" { 93 | export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1) 94 | export SLEEP_AS_RELAY=1 95 | TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest5.rb || true & 96 | sleep 0.1 97 | TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest5.rb 98 | wait 99 | 100 | assert_status 0 101 | assert_output_contains "Starting test-queue master" 102 | } 103 | 104 | @test "multi-master remote master succeeds when all tests pass" { 105 | export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1) 106 | export SLEEP_AS_MASTER=1 107 | TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest5.rb || true & 108 | sleep 0.1 109 | TEST_QUEUE_RELAY=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest5.rb 110 | wait 111 | 112 | assert_status 0 113 | assert_output_contains "Starting test-queue master" 114 | } 115 | 116 | @test "multi-master central master fails when a test fails" { 117 | export FAIL=1 118 | export SLEEP_AS_RELAY=1 119 | export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1) 120 | TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest5.rb || true & 121 | sleep 0.1 122 | TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest5.rb 123 | wait 124 | 125 | assert_status 1 126 | assert_output_contains "Starting test-queue master" 127 | assert_output_contains "1) Failure:" 128 | assert_output_contains "MinitestFailure#test_fail" 129 | } 130 | 131 | @test "multi-master remote master fails when a test fails" { 132 | export FAIL=1 133 | export SLEEP_AS_MASTER=1 134 | export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1) 135 | TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest5.rb || true & 136 | sleep 0.1 137 | TEST_QUEUE_RELAY=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest5.rb 138 | wait 139 | 140 | assert_status 1 141 | assert_output_contains "Starting test-queue master" 142 | assert_output_contains "1) Failure:" 143 | assert_output_contains "MinitestFailure#test_fail" 144 | } 145 | 146 | @test "multi-master central master prints out remote master messages" { 147 | export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1) 148 | TEST_QUEUE_RELAY=0.0.0.0:12345 TEST_QUEUE_REMOTE_MASTER_MESSAGE="hello from remote master" bundle exec minitest-queue ./test/examples/example_minitest5.rb & 149 | TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec minitest-queue ./test/examples/example_minitest5.rb 150 | wait 151 | 152 | assert_status 0 153 | assert_output_contains "hello from remote master" 154 | } 155 | 156 | @test "recovers from child processes dying in an unorderly way" { 157 | export KILL=1 158 | run bundle exec minitest-queue ./test/examples/example_minitest5.rb 159 | assert_status 1 160 | assert_output_contains "SIGKILL (signal 9)" 161 | } 162 | 163 | @test "minitest-queue fails when TEST_QUEUE_WORKERS is <= 0" { 164 | export TEST_QUEUE_WORKERS=0 165 | run bundle exec minitest-queue ./test/examples/example_minitest5.rb 166 | assert_status 1 167 | assert_output_contains "Worker count (0) must be greater than 0" 168 | } 169 | 170 | @test "minitest-queue fails when given a missing test file" { 171 | run bundle exec minitest-queue ./test/examples/does_not_exist.rb 172 | assert_status 1 173 | assert_output_contains "Aborting: Discovering suites failed" 174 | } 175 | 176 | @test "minitest-queue fails when given a malformed test file" { 177 | [ -f README.md ] 178 | run bundle exec minitest-queue README.md 179 | assert_status 1 180 | assert_output_contains "Aborting: Discovering suites failed" 181 | } 182 | 183 | @test "minitest-queue handles test file being deleted" { 184 | cp test/examples/example_mini{test5,spec}.rb $SCRATCH 185 | 186 | run bundle exec minitest-queue $SCRATCH/* 187 | assert_status 0 188 | assert_output_contains "Meme::when asked about blending possibilities" 189 | 190 | rm $SCRATCH/example_minispec.rb 191 | 192 | run bundle exec minitest-queue $SCRATCH/* 193 | assert_status 0 194 | refute_output_contains "Meme::when asked about blending possibilities" 195 | } 196 | 197 | @test "minitest-queue handles suites changing inside a file" { 198 | cp test/examples/example_minispec.rb $SCRATCH 199 | 200 | run bundle exec minitest-queue $SCRATCH/example_minispec.rb 201 | assert_status 0 202 | assert_output_contains "Meme::when asked about blending possibilities" 203 | 204 | sed -i'' -e 's/Meme/Meme2/g' $SCRATCH/example_minispec.rb 205 | 206 | run bundle exec minitest-queue $SCRATCH/example_minispec.rb 207 | assert_status 0 208 | assert_output_contains "Meme2::when asked about blending possibilities" 209 | } 210 | -------------------------------------------------------------------------------- /test/rspec3.bats: -------------------------------------------------------------------------------- 1 | load "testlib" 2 | 3 | setup() { 4 | require_gem "rspec" "~> 3.0" 5 | } 6 | 7 | @test "rspec-queue succeeds when all specs pass" { 8 | run bundle exec rspec-queue ./test/examples/example_spec.rb 9 | assert_status 0 10 | assert_output_contains "Randomized with seed" 11 | assert_output_contains "Starting test-queue master" 12 | assert_output_contains "16 examples, 0 failures" 13 | assert_output_contains "16 examples, 0 failures" 14 | } 15 | 16 | @test "rspec-queue succeeds all specs pass in the default spec directory even if directory path is omitted" { 17 | run bundle exec rspec-queue 18 | assert_status 0 19 | assert_output_contains "Starting test-queue master" 20 | assert_output_contains "6 examples, 0 failures" 21 | assert_output_contains "0 examples, 0 failures" 22 | } 23 | 24 | @test "rspec-queue fails when a spec fails" { 25 | export FAIL=1 26 | run bundle exec rspec-queue ./test/examples/example_spec.rb 27 | assert_status 1 28 | assert_output_contains "1) RSpecFailure fails" 29 | assert_output_contains "RSpecFailure fails" 30 | assert_output_contains "expected: :bar" 31 | assert_output_contains "got: :foo" 32 | } 33 | 34 | @test "TEST_QUEUE_SPLIT_GROUPS splits splittable groups" { 35 | export TEST_QUEUE_SPLIT_GROUPS=true 36 | run bundle exec rspec-queue ./test/examples/example_split_spec.rb 37 | assert_status 0 38 | 39 | assert_output_matches '\[ 1\] +1 example, 0 failures' 40 | assert_output_matches '\[ 2\] +1 example, 0 failures' 41 | } 42 | 43 | @test "TEST_QUEUE_SPLIT_GROUPS does not split unsplittable groups" { 44 | export TEST_QUEUE_SPLIT_GROUPS=true 45 | export NOSPLIT=1 46 | run bundle exec rspec-queue ./test/examples/example_split_spec.rb 47 | assert_status 0 48 | 49 | assert_output_contains "2 examples, 0 failures" 50 | assert_output_contains "0 examples, 0 failures" 51 | } 52 | 53 | @test "rspec-queue supports shared example groups" { 54 | run bundle exec rspec-queue ./test/examples/example_use_shared_example1_spec.rb \ 55 | ./test/examples/example_use_shared_example2_spec.rb 56 | assert_status 0 57 | } 58 | -------------------------------------------------------------------------------- /test/rspec4.bats: -------------------------------------------------------------------------------- 1 | load "testlib" 2 | 3 | setup() { 4 | require_gem "rspec" "4.0.0.pre" 5 | } 6 | 7 | @test "rspec-queue succeeds when all specs pass" { 8 | run bundle exec rspec-queue ./test/examples/example_spec.rb 9 | assert_status 0 10 | assert_output_contains "Randomized with seed" 11 | assert_output_contains "Starting test-queue master" 12 | assert_output_contains "16 examples, 0 failures" 13 | assert_output_contains "16 examples, 0 failures" 14 | } 15 | 16 | @test "rspec-queue succeeds all specs pass in the default spec directory even if directory path is omitted" { 17 | run bundle exec rspec-queue 18 | assert_status 0 19 | assert_output_contains "Starting test-queue master" 20 | assert_output_contains "6 examples, 0 failures" 21 | assert_output_contains "0 examples, 0 failures" 22 | } 23 | 24 | @test "rspec-queue fails when a spec fails" { 25 | export FAIL=1 26 | run bundle exec rspec-queue ./test/examples/example_spec.rb 27 | assert_status 1 28 | assert_output_contains "1) RSpecFailure fails" 29 | assert_output_contains "RSpecFailure fails" 30 | assert_output_contains "expected: :bar" 31 | assert_output_contains "got: :foo" 32 | } 33 | 34 | @test "TEST_QUEUE_SPLIT_GROUPS splits splittable groups" { 35 | export TEST_QUEUE_SPLIT_GROUPS=true 36 | run bundle exec rspec-queue ./test/examples/example_split_spec.rb 37 | assert_status 0 38 | 39 | assert_output_matches '\[ 1\] +1 example, 0 failures' 40 | assert_output_matches '\[ 2\] +1 example, 0 failures' 41 | } 42 | 43 | @test "TEST_QUEUE_SPLIT_GROUPS does not split unsplittable groups" { 44 | export TEST_QUEUE_SPLIT_GROUPS=true 45 | export NOSPLIT=1 46 | run bundle exec rspec-queue ./test/examples/example_split_spec.rb 47 | assert_status 0 48 | 49 | assert_output_contains "2 examples, 0 failures" 50 | assert_output_contains "0 examples, 0 failures" 51 | } 52 | 53 | @test "rspec-queue supports shared example groups" { 54 | run bundle exec rspec-queue ./test/examples/example_use_shared_example1_spec.rb \ 55 | ./test/examples/example_use_shared_example2_spec.rb 56 | assert_status 0 57 | } 58 | -------------------------------------------------------------------------------- /test/sleepy_runner.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | $LOAD_PATH.unshift File.expand_path('../lib', __dir__) 4 | 5 | require 'test_queue' 6 | require 'test_queue/runner/minitest' 7 | 8 | class SleepyTestRunner < TestQueue::Runner::Minitest 9 | def after_fork(_num) 10 | if ENV['SLEEP_AS_RELAY'] && relay? || ENV['SLEEP_AS_MASTER'] && !relay? 11 | sleep 5 12 | end 13 | end 14 | end 15 | 16 | SleepyTestRunner.new.execute 17 | -------------------------------------------------------------------------------- /test/testlib.bash: -------------------------------------------------------------------------------- 1 | # Skip this test unless the bundle contains a gem matching the required 2 | # version. Example: 3 | # 4 | # require_gem "minitest" "~> 5.3" 5 | require_gem() { 6 | name=$1 7 | requirement=$2 8 | 9 | set +e 10 | version=$(bundle exec ruby - <= 3.0" 5 | } 6 | 7 | @test "testunit-queue succeeds when all tests pass" { 8 | run bundle exec testunit-queue ./test/examples/*_testunit.rb 9 | assert_status 0 10 | assert_output_contains "Starting test-queue master" 11 | } 12 | 13 | @test "testunit-queue fails when a test fails" { 14 | export FAIL=1 15 | run bundle exec testunit-queue ./test/examples/*_testunit.rb 16 | assert_status 1 17 | assert_output_contains "Starting test-queue master" 18 | assert_output_contains "Failure:" 19 | assert_output_contains "test_fail(TestUnitFailure)" 20 | } 21 | 22 | @test "TEST_QUEUE_SPLIT_GROUPS testunit-queue splits splittable sub_test_cases" { 23 | export TEST_QUEUE_SPLIT_GROUPS=true 24 | run bundle exec testunit-queue ./test/examples/example_testunit_split.rb 25 | assert_status 0 26 | 27 | assert_output_matches '\[ 1\] +2 tests,' 28 | assert_output_matches '\[ 2\] +2 tests,' 29 | } 30 | 31 | @test "TEST_QUEUE_SPLIT_GROUPS does not testunit-queue splits splittable sub_test_cases" { 32 | export TEST_QUEUE_SPLIT_GROUPS=true 33 | export NOSPLIT=true 34 | run bundle exec testunit-queue ./test/examples/example_testunit_split.rb 35 | assert_status 0 36 | 37 | assert_output_matches '\[ .\] +1 tests,' 38 | assert_output_matches '\[ .\] +3 tests,' 39 | } 40 | -------------------------------------------------------------------------------- /test/turnip.bats: -------------------------------------------------------------------------------- 1 | load "testlib" 2 | 3 | setup() { 4 | require_gem "turnip" "~> 4.4" 5 | } 6 | 7 | @test "rspec-queue succeeds when all features pass" { 8 | run bundle exec rspec-queue ./test/examples/features -r turnip/rspec -r ./test/examples/features/turnip_steps/global_steps 9 | assert_status 0 10 | assert_output_contains "Starting test-queue master" 11 | } 12 | 13 | @test "rspec-queue succeeds when all features and specs pass" { 14 | run bundle exec rspec-queue ./test/examples -r turnip/rspec -r ./test/examples/features/turnip_steps/global_steps 15 | assert_status 0 16 | assert_output_contains "Starting test-queue master" 17 | } 18 | 19 | @test "rspec-queue fails when a feature fails" { 20 | export FAIL=1 21 | run bundle exec rspec-queue test/examples/features -r turnip/rspec -r ./test/examples/features/turnip_steps/global_steps 22 | refute_status 0 23 | assert_output_contains "Starting test-queue master" 24 | assert_output_contains "test/examples/features/bad.feature:4" 25 | assert_output_contains "test/examples/features/example2.feature:28" 26 | } 27 | 28 | @test "rspec-queue fails when given a missing feature" { 29 | run bundle exec rspec-queue test/examples/does_not_exist.feature -r turnip/rspec -r ./test/examples/features/turnip_steps/global_steps 30 | refute_status 0 31 | assert_output_contains "Aborting: Discovering suites failed." 32 | } 33 | 34 | @test "rspec-queue fails when given a malformed feature" { 35 | [ -f README.md ] 36 | run bundle exec rspec-queue README.md -r turnip/rspec -r ./test/examples/features/turnip_steps/global_steps 37 | 38 | refute_status 0 39 | assert_output_matches 'Aborting: Discovering suites failed\.|README\.md: Parser errors:' 40 | } 41 | --------------------------------------------------------------------------------