├── .gitignore ├── .github ├── CODEOWNERS ├── boomper.yml ├── workflows │ └── release-drafter.yml └── release-drafter.yml ├── ruby ├── Rakefile ├── tests │ ├── no-test-failures │ │ ├── junit-2.xml │ │ ├── junit-1.xml │ │ └── junit-3.xml │ ├── .tests-in-hidden-dir │ │ └── junit-1.xml │ ├── skipped-test │ │ └── junit.xml │ ├── empty-failure-body │ │ └── junit.xml │ ├── failure-with-cdata │ │ └── junit.xml │ ├── two-test-failures │ │ ├── junit-2.xml │ │ ├── junit-1.xml │ │ └── junit-3.xml │ ├── test-failure-and-error │ │ ├── junit-2.xml │ │ ├── junit-1.xml │ │ └── junit-3.xml │ ├── tests-in-sub-dirs │ │ └── sub-dir │ │ │ ├── junit-2.xml │ │ │ ├── junit-1.xml │ │ │ └── junit-3.xml │ ├── custom-job-uuid-pattern │ │ └── junit-123-456-custom-pattern.xml │ ├── missing-message-attribute │ │ └── junit.xml │ └── annotate_test.rb └── bin │ └── annotate ├── renovate.json ├── docker-compose.yml ├── tests ├── 2-tests-1-failure.output ├── 2-slowest-tests.output └── command.bats ├── .buildkite └── pipeline.yml ├── plugin.yml ├── LICENSE ├── hooks └── command ├── CODE_OF_CONDUCT.md └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /tests/tmp -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @buildkite-plugins/plugin-engineering 2 | -------------------------------------------------------------------------------- /.github/boomper.yml: -------------------------------------------------------------------------------- 1 | # Config for https://github.com/apps/boomper-bot 2 | updates: 3 | - path: README.md 4 | pattern: 'junit-annotate#(.*):' -------------------------------------------------------------------------------- /ruby/Rakefile: -------------------------------------------------------------------------------- 1 | require "rake/testtask" 2 | 3 | Rake::TestTask.new do |t| 4 | t.test_files = FileList['tests/**/*_test.rb'] 5 | end 6 | 7 | task default: :test -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base", 4 | ":disableDependencyDashboard" 5 | ], 6 | "bundler": { 7 | "enabled": true 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | ruby: 4 | image: ruby:3.4-alpine@sha256:ab15990e12099ce2a8782cbf66fd7d1be800678c8c7c0d350400c2b992ed54f8 5 | command: rake 6 | working_dir: /src 7 | volumes: 8 | - "./ruby:/src" -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | workflow_dispatch: 9 | 10 | jobs: 11 | update_release_draft: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: release-drafter/release-drafter@v6 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | -------------------------------------------------------------------------------- /tests/2-tests-1-failure.output: -------------------------------------------------------------------------------- 1 | Failures: 0 2 | Errors: 1 3 | Skipped: 0 4 | Total tests: 2 5 | 6 |
7 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec 8 | 9 |

expected: 250 got: 500 (compared using eql?)

10 | 11 | 
First line of failure output
12 |       Second line of failure output
13 | 14 |
15 | -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | # Config for https://github.com/apps/release-drafter 2 | name-template: v$NEXT_MINOR_VERSION (🐣 Code Name) 3 | tag-template: v$NEXT_MINOR_VERSION 4 | template: | 5 | ## TODO: Added/Removed/Fixed/Changed 6 | 7 | $CHANGES 8 | 9 | ## Upgrading 10 | 11 | To upgrade, update your `pipeline.yml` files: 12 | 13 | ```diff 14 | steps: 15 | - plugins: 16 | - junit-annotate#$PREVIOUS_TAG: 17 | + junit-annotate#TODO: 18 | ``` 19 | -------------------------------------------------------------------------------- /ruby/tests/no-test-failures/junit-2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /ruby/tests/no-test-failures/junit-1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /ruby/tests/.tests-in-hidden-dir/junit-1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /ruby/tests/skipped-test/junit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.buildkite/pipeline.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - label: ":bash: Plugin" 3 | plugins: 4 | - plugin-tester#v1.3.0: ~ 5 | 6 | - label: ":ruby: Ruby" 7 | plugins: 8 | - docker-compose#v5.12.1: 9 | run: ruby 10 | 11 | - label: "✨ Lint" 12 | plugins: 13 | - plugin-linter#v3.3.0: 14 | id: junit-annotate 15 | 16 | - label: ":bash: Shellcheck" 17 | plugins: 18 | - shellcheck#v1.4.0: 19 | files: hooks/* 20 | 21 | - label: "Security Scan" 22 | key: security_scan 23 | plugins: 24 | - secrets#v1.0.2: 25 | variables: 26 | GITHUB_TOKEN: GITHUB_TOKEN 27 | - ossf-scorecard#v1.0.1: 28 | github_token: $$GITHUB_TOKEN 29 | -------------------------------------------------------------------------------- /ruby/tests/empty-failure-body/junit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /ruby/tests/failure-with-cdata/junit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /plugin.yml: -------------------------------------------------------------------------------- 1 | name: Junit Annotate 2 | description: Annotates your build using JUnit XML reports 3 | author: https://github.com/buildkite 4 | requirements: 5 | - docker 6 | configuration: 7 | properties: 8 | artifacts: 9 | type: string 10 | always-annotate: 11 | type: boolean 12 | context: 13 | type: string 14 | failure-format: 15 | type: string 16 | enum: 17 | - classname 18 | - file 19 | fail-build-on-error: 20 | type: boolean 21 | failed-download-exit-code: 22 | type: integer 23 | job-uuid-file-pattern: 24 | type: string 25 | min-tests: 26 | type: integer 27 | report-skipped: 28 | type: boolean 29 | report-slowest: 30 | type: integer 31 | ruby-image: 32 | type: string 33 | run-in-docker: 34 | type: boolean 35 | required: 36 | - artifacts 37 | additionalProperties: false 38 | -------------------------------------------------------------------------------- /tests/2-slowest-tests.output: -------------------------------------------------------------------------------- 1 | Failures: 0 2 | Errors: 0 3 | Skipped: 0 4 | Total tests: 8 5 | 6 |
7 | 5 slowest tests 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
UnitTestTime
spec.models.account_specAccount#maximum_jobs_added_by_pipeline_changer returns 250 by default0.977127
spec.models.account_specAccount#maximum_jobs_added_by_pipeline_changer returns 250 by default0.967127
spec.models.account_specAccount#maximum_jobs_added_by_pipeline_changer returns 500 if the account is ABC0.620013
spec.models.account_specAccount#maximum_jobs_added_by_pipeline_changer returns 900 if the account is F000.520013
spec.models.account_specAccount#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ0.420013
18 |
-------------------------------------------------------------------------------- /ruby/tests/two-test-failures/junit-2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 5 | 6 | expected: 700 7 | got: 500 8 | 9 | (compared using eql?) 10 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 11 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 12 | ./spec/support/log.rb:17:in `run' 13 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 14 | 15 | -------------------------------------------------------------------------------- /ruby/tests/test-failure-and-error/junit-2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 5 | 6 | expected: 700 7 | got: 500 8 | 9 | (compared using eql?) 10 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 11 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 12 | ./spec/support/log.rb:17:in `run' 13 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 14 | 15 | -------------------------------------------------------------------------------- /ruby/tests/tests-in-sub-dirs/sub-dir/junit-2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 5 | 6 | expected: 700 7 | got: 500 8 | 9 | (compared using eql?) 10 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 11 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 12 | ./spec/support/log.rb:17:in `run' 13 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 14 | 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Buildkite 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /ruby/tests/no-test-failures/junit-3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /ruby/tests/test-failure-and-error/junit-1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 6 | 7 | expected: 250 8 | got: 500 9 | 10 | (compared using eql?) 11 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 12 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 13 | ./spec/support/log.rb:17:in `run' 14 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 15 | 16 | -------------------------------------------------------------------------------- /ruby/tests/two-test-failures/junit-1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 6 | 7 | expected: 250 8 | got: 500 9 | 10 | (compared using eql?) 11 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 12 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 13 | ./spec/support/log.rb:17:in `run' 14 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 15 | 16 | -------------------------------------------------------------------------------- /ruby/tests/tests-in-sub-dirs/sub-dir/junit-1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 6 | 7 | expected: 250 8 | got: 500 9 | 10 | (compared using eql?) 11 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 12 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 13 | ./spec/support/log.rb:17:in `run' 14 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 15 | 16 | -------------------------------------------------------------------------------- /ruby/tests/custom-job-uuid-pattern/junit-123-456-custom-pattern.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 6 | 7 | expected: 250 8 | got: 500 9 | 10 | (compared using eql?) 11 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 12 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 13 | ./spec/support/log.rb:17:in `run' 14 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 15 | 16 | -------------------------------------------------------------------------------- /ruby/tests/missing-message-attribute/junit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /ruby/tests/test-failure-and-error/junit-3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 6 | 7 | expected: 700 8 | got: 500 9 | 10 | (compared using eql?) 11 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 12 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 13 | ./spec/support/log.rb:17:in `run' 14 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 15 | 16 | 17 | 18 | 19 | 20 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 21 | 22 | expected: 250 23 | got: 500 24 | 25 | (compared using eql?) 26 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 27 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 28 | ./spec/support/log.rb:17:in `run' 29 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /ruby/tests/two-test-failures/junit-3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 6 | 7 | expected: 700 8 | got: 500 9 | 10 | (compared using eql?) 11 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 12 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 13 | ./spec/support/log.rb:17:in `run' 14 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 15 | 16 | 17 | 18 | 19 | 20 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 21 | 22 | expected: 250 23 | got: 500 24 | 25 | (compared using eql?) 26 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 27 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 28 | ./spec/support/log.rb:17:in `run' 29 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /ruby/tests/tests-in-sub-dirs/sub-dir/junit-3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 6 | 7 | expected: 700 8 | got: 500 9 | 10 | (compared using eql?) 11 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 12 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 13 | ./spec/support/log.rb:17:in `run' 14 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 15 | 16 | 17 | 18 | 19 | 20 | Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250) 21 | 22 | expected: 250 23 | got: 500 24 | 25 | (compared using eql?) 26 | ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>' 27 | ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>' 28 | ./spec/support/log.rb:17:in `run' 29 | ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /ruby/bin/annotate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'rexml/document' 4 | require 'rexml/element' 5 | require 'cgi/util' 6 | 7 | # Reads a list of junit files and returns a nice Buildkite build annotation on 8 | # STDOUT that summarizes any failures. 9 | 10 | junits_dir = ARGV[0] 11 | abort("Usage: annotate ") unless junits_dir 12 | abort("#{junits_dir} does not exist") unless Dir.exist?(junits_dir) 13 | 14 | job_pattern = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN'] 15 | job_pattern = '-(.*).xml' if !job_pattern || job_pattern.empty? 16 | 17 | failure_format = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT'] 18 | failure_format = 'classname' if !failure_format || failure_format.empty? 19 | 20 | report_slowest = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST'].to_i 21 | report_skipped = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED'] == 'true' 22 | 23 | class Failure < Struct.new(:name, :unit_name, :body, :job, :message) 24 | end 25 | 26 | class Timing < Struct.new(:name, :unit_name, :time) 27 | end 28 | 29 | junit_report_files = Dir.glob(File.join(junits_dir, "**", "*"), File::FNM_DOTMATCH) 30 | testcases = 0 31 | tests = { 32 | failure: [], 33 | error: [], 34 | skipped: [] 35 | } 36 | timings = [] 37 | 38 | def text_content(element) 39 | # Handle mulptiple CDATA/text children elements 40 | text = element.texts().map(&:value).join.strip 41 | if text.empty? 42 | nil 43 | else 44 | text 45 | end 46 | end 47 | 48 | def message_content(element) 49 | # Handle empty attributes 50 | message = element.attributes['message']; 51 | if message.nil? || message.empty? 52 | nil 53 | else 54 | message.to_s 55 | end 56 | end 57 | 58 | junit_report_files.sort.each do |file| 59 | next if File.directory?(file) 60 | 61 | STDERR.puts "Parsing #{file.sub(junits_dir, '')}" 62 | job = File.basename(file)[/#{job_pattern}/, 1] 63 | xml = File.read(file) 64 | doc = REXML::Document.new(xml) 65 | 66 | REXML::XPath.each(doc, '//testsuite/testcase') do |testcase| 67 | testcases += 1 68 | name = testcase.attributes['name'].to_s 69 | unit_name = testcase.attributes[failure_format].to_s 70 | time = testcase.attributes['time'].to_f 71 | timings << Timing.new(name, unit_name, time) 72 | testcase.elements.each("failure | error | skipped") do |elem| 73 | tests[elem.name.to_sym] << Failure.new(name, unit_name, text_content(elem), job, message_content(elem)) 74 | end 75 | end 76 | end 77 | 78 | STDERR.puts "--- ✍️ Preparing annotation" 79 | 80 | puts "Failures: #{tests[:failure].length}" 81 | puts "Errors: #{tests[:error].length}" 82 | puts "Skipped: #{tests[:skipped].length}" 83 | puts "Total tests: #{testcases}" 84 | 85 | skipped = tests.delete(:skipped) # save value for later 86 | 87 | tests.values.flatten.each do |failure| 88 | puts "" 89 | puts "
" 90 | puts "#{CGI.escapeHTML failure.name} in #{CGI.escapeHTML failure.unit_name}\n\n" 91 | if failure.message 92 | puts "

#{CGI.escapeHTML failure.message.chomp.strip}

\n\n" 93 | end 94 | if failure.body 95 | puts "
#{CGI.escapeHTML(failure.body.chomp.strip)}
\n\n" 96 | end 97 | if failure.job 98 | puts "in Job ##{failure.job}" 99 | end 100 | puts "
" 101 | end 102 | 103 | if report_slowest > 0 104 | STDERR.puts "Reporting slowest tests ⏱" 105 | puts "" 106 | puts "
" 107 | puts "#{report_slowest} slowest tests\n\n" 108 | puts "" 109 | puts "" 110 | puts "" 111 | timings.sort_by(&:time).reverse.take(report_slowest).each do |timing| 112 | puts "" 113 | end 114 | puts "" 115 | puts "
UnitTestTime
#{timing.unit_name}#{timing.name}#{timing.time}
" 116 | puts "
" 117 | end 118 | 119 | if report_skipped 120 | STDERR.puts "Reporting skipped tests" 121 | puts "" 122 | puts "
" 123 | puts "#{skipped.length} tests skipped\n\n" 124 | puts "
    " 125 | skipped.each do |sk| 126 | puts "
  1. #{CGI.escapeHTML sk.name} in #{CGI.escapeHTML sk.unit_name} (#{CGI.escapeHTML sk.message || "no reason"})
    2. \n" 127 | end 128 | puts "
" 129 | puts "
" 130 | end 131 | 132 | exit 64 if tests.values.flatten.any? # special exit code to signal test failures 133 | -------------------------------------------------------------------------------- /hooks/command: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS:-}" ]]; then 6 | echo "🚨 Missing artifacts configuration for the plugin" 7 | exit 1 8 | fi 9 | 10 | PLUGIN_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)/.." 11 | MAX_SIZE=1024 # in KB 12 | RUBY_IMAGE="${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_RUBY_IMAGE:-ruby:3.1-alpine@sha256:a39e26d0598837f08c75a42c8b0886d9ed5cc862c4b535662922ee1d05272fca}" 13 | 14 | artifacts_dir="$(pwd)/$(mktemp -d "junit-annotate-plugin-artifacts-tmp.XXXXXXXXXX")" 15 | annotation_dir="$(pwd)/$(mktemp -d "junit-annotate-plugin-annotation-tmp.XXXXXXXXXX")" 16 | annotation_path="${annotation_dir}/annotation.md" 17 | annotation_style="info" 18 | fail_build=0 19 | has_errors=0 20 | create_annotation=0 21 | 22 | # shellcheck disable=2317 # this is a signal function 23 | # shellcheck disable=2329 # this function is used but SC isn't picking it up 24 | function cleanup { 25 | rm -rf "${artifacts_dir}" 26 | rm -rf "${annotation_dir}" 27 | } 28 | 29 | function check_size { 30 | local size_in_kb 31 | size_in_kb=$(du -k "${annotation_path}" | cut -f 1) 32 | [ "${size_in_kb}" -lt "${MAX_SIZE}" ] 33 | } 34 | 35 | trap cleanup EXIT 36 | 37 | echo "--- :junit: Download the junits" 38 | if ! buildkite-agent artifact download "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS}" "$artifacts_dir"; then 39 | echo "--- :boom: Could not download artifacts" 40 | exit "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILED_DOWNLOAD_EXIT_CODE:-2}" 41 | fi 42 | 43 | echo "--- :junit: Processing the junits" 44 | 45 | set +e 46 | if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_RUN_IN_DOCKER:-true}" =~ "true" ]]; then 47 | docker \ 48 | --log-level "error" \ 49 | run \ 50 | --rm \ 51 | --volume "$artifacts_dir:/junits:Z" \ 52 | --volume "$PLUGIN_DIR/ruby:/src:Z" \ 53 | --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN:-}" \ 54 | --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT:-}" \ 55 | --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST:-}" \ 56 | --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED:-}" \ 57 | "${RUBY_IMAGE}" ruby /src/bin/annotate /junits \ 58 | > "$annotation_path" 59 | else 60 | ruby "${PLUGIN_DIR}/ruby/bin/annotate" "${artifacts_dir}" > "$annotation_path" 61 | fi 62 | 63 | exit_code=$? 64 | set -e 65 | 66 | if [[ $exit_code -eq 64 ]]; then # special exit code to signal test failures 67 | has_errors=1 68 | create_annotation=1 69 | annotation_style="error" 70 | if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR:-false}" =~ (true|on|1) ]]; then 71 | echo "--- :boom: Build will fail due to errors being found" 72 | fail_build=1 73 | fi 74 | elif [[ $exit_code -ne 0 ]]; then 75 | echo "--- :boom: Error when processing JUnit tests" 76 | exit $exit_code 77 | fi 78 | 79 | cat "$annotation_path" 80 | 81 | if [ $has_errors -eq 0 ]; then 82 | # done in nested if to simplify outer conditions 83 | if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ALWAYS_ANNOTATE:-false}" =~ (true|on|1) ]]; then 84 | echo "Will create annotation anyways" 85 | create_annotation=1 86 | fi 87 | 88 | if [[ -n "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST:-}" ]]; then 89 | echo "Create annotation with slowest tests" 90 | create_annotation=1 91 | fi 92 | 93 | if [[ -e "${annotation_path}" ]]; then 94 | TOTAL_TESTS=$(head -5 "${annotation_path}" | grep 'Total tests' | cut -d\ -f3) 95 | else 96 | TOTAL_TESTS=0 97 | fi 98 | 99 | if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS:-0}" -gt "${TOTAL_TESTS}" ]]; then 100 | create_annotation=1 101 | fail_build=1 102 | echo ":warning: Less than ${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS} tests analyzed" 103 | fi 104 | elif ! check_size; then 105 | echo "--- :warning: Failures too large to annotate" 106 | 107 | # creating a simplified version of the annotation 108 | mv "${annotation_path}" "${annotation_path}2" 109 | head -5 "${annotation_path}2" >"${annotation_path}" 110 | # || true is to avoid issues if no summary is found 111 | grep '' "${annotation_path}2" >>"${annotation_path}" || true 112 | 113 | if ! check_size; then 114 | echo "The failures are too large to create a build annotation. Please inspect the failed JUnit artifacts manually." 115 | create_annotation=0 116 | else 117 | echo "The failures are too large to create complete annotation, using a simplified annotation" 118 | fi 119 | fi 120 | 121 | if [ $create_annotation -ne 0 ]; then 122 | echo "--- :buildkite: Creating annotation" 123 | # shellcheck disable=SC2002 124 | cat "$annotation_path" | buildkite-agent annotate --context "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_CONTEXT:-junit}" --style "$annotation_style" 125 | fi 126 | 127 | exit $fail_build 128 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, caste, color, religion, or sexual 10 | identity and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the overall 26 | community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or advances of 31 | any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email address, 35 | without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at [coc@buildkite.com](mailto:coc@buildkite.com) 63 | All complaints will be reviewed and investigated promptly and fairly. 64 | 65 | All community leaders are obligated to respect the privacy and security of the 66 | reporter of any incident. 67 | 68 | ## Enforcement Guidelines 69 | 70 | Community leaders will follow these Community Impact Guidelines in determining 71 | the consequences for any action they deem in violation of this Code of Conduct: 72 | 73 | ### 1. Correction 74 | 75 | **Community Impact**: Use of inappropriate language or other behavior deemed 76 | unprofessional or unwelcome in the community. 77 | 78 | **Consequence**: A private, written warning from community leaders, providing 79 | clarity around the nature of the violation and an explanation of why the 80 | behavior was inappropriate. A public apology may be requested. 81 | 82 | ### 2. Warning 83 | 84 | **Community Impact**: A violation through a single incident or series of 85 | actions. 86 | 87 | **Consequence**: A warning with consequences for continued behavior. No 88 | interaction with the people involved, including unsolicited interaction with 89 | those enforcing the Code of Conduct, for a specified period of time. This 90 | includes avoiding interactions in community spaces as well as external channels 91 | like social media. Violating these terms may lead to a temporary or permanent 92 | ban. 93 | 94 | ### 3. Temporary Ban 95 | 96 | **Community Impact**: A serious violation of community standards, including 97 | sustained inappropriate behavior. 98 | 99 | **Consequence**: A temporary ban from any sort of interaction or public 100 | communication with the community for a specified period of time. No public or 101 | private interaction with the people involved, including unsolicited interaction 102 | with those enforcing the Code of Conduct, is allowed during this period. 103 | Violating these terms may lead to a permanent ban. 104 | 105 | ### 4. Permanent Ban 106 | 107 | **Community Impact**: Demonstrating a pattern of violation of community 108 | standards, including sustained inappropriate behavior, harassment of an 109 | individual, or aggression toward or disparagement of classes of individuals. 110 | 111 | **Consequence**: A permanent ban from any sort of public interaction within the 112 | community. 113 | 114 | ## Attribution 115 | 116 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 117 | version 2.1, available at 118 | [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. 119 | 120 | Community Impact Guidelines were inspired by 121 | [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. 122 | 123 | For answers to common questions about this code of conduct, see the FAQ at 124 | [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at 125 | [https://www.contributor-covenant.org/translations][translations]. 126 | 127 | [homepage]: https://www.contributor-covenant.org 128 | [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html 129 | [Mozilla CoC]: https://github.com/mozilla/diversity 130 | [FAQ]: https://www.contributor-covenant.org/faq 131 | [translations]: https://www.contributor-covenant.org/translations 132 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JUnit Annotate Buildkite Plugin [![Build status](https://badge.buildkite.com/e57701b1037f2c77d0b3f2e4901559ed2e8f131119cd7806ad.svg?branch=master)](https://buildkite.com/buildkite/plugins-junit-annotate) 2 | 3 | A [Buildkite plugin](https://buildkite.com/docs/agent/v3/plugins) that parses junit.xml artifacts (generated across any number of parallel steps) and creates a [build annotation](https://buildkite.com/docs/agent/v3/cli-annotate) listing the individual tests that failed. 4 | 5 | ## Example 6 | 7 | The following pipeline will run `test.sh` jobs in parallel, and then process all the resulting JUnit XML files to create a summary build annotation. 8 | 9 | ```yml 10 | steps: 11 | - command: test.sh 12 | parallelism: 50 13 | artifact_paths: tmp/junit-*.xml 14 | - wait: ~ 15 | continue_on_failure: true 16 | - plugins: 17 | - junit-annotate#v2.7.0: 18 | artifacts: tmp/junit-*.xml 19 | ``` 20 | 21 | For scenarios where you have different artifact paths that you want to add as annotation then call the plugin multiple times in the pipeline with different contexts as shown below: 22 | 23 | ```yml 24 | steps: 25 | - command: test.sh 26 | parallelism: 50 27 | artifact_paths: tmp/junit-*.xml 28 | - command: anothertest.sh 29 | artifact_paths: artifacts/junit-*.xml 30 | - wait: ~ 31 | continue_on_failure: true 32 | - plugins: 33 | - junit-annotate#v2.7.0: 34 | artifacts: tmp/junit-*.xml 35 | - plugins: 36 | - junit-annotate#v2.7.0: 37 | artifacts: artifacts/junit-*.xml 38 | context: junit-artifacts 39 | ``` 40 | 41 | ## Configuration 42 | 43 | ### `artifacts` (required) 44 | 45 | The artifact glob path to find the JUnit XML files. 46 | 47 | Example: `tmp/junit-*.xml` 48 | 49 | ### `always-annotate` (optional, boolean) 50 | 51 | Forces the creation of the annotation even when no failures or errors are found 52 | 53 | ### `context` (optional) 54 | 55 | Default: `junit` 56 | 57 | The buildkite annotation context to use. Useful to differentiate multiple runs of this plugin in a single pipeline. 58 | 59 | ### `job-uuid-file-pattern` (optional) 60 | 61 | Default: `-(.*).xml` 62 | 63 | The regular expression (with capture group) that matches the job UUID in the junit file names. This is used to create the job links in the annotation. 64 | 65 | To use this, configure your test reporter to embed the `$BUILDKITE_JOB_ID` environment variable into your junit file names. For example `"junit-buildkite-job-$BUILDKITE_JOB_ID.xml"`. 66 | 67 | ### `failure-format` (optional) 68 | 69 | This setting controls the format of your failed test in the main annotation summary. 70 | 71 | There are two options for this: 72 | * `classname` (the default) 73 | * displays: `MyClass::UnderTest text of the failed expectation in path.to.my_class.under_test` 74 | * `file` 75 | * displays: `MyClass::UnderTest text of the failed expectation in path/to/my_class/under_test.file_ext` 76 | 77 | ### `fail-build-on-error` (optional) 78 | 79 | Default: `false` 80 | 81 | If this setting is true and any errors are found in the JUnit XML files during parsing, the annotation step will exit with a non-zero value, which should cause the build to fail. 82 | 83 | ### `failed-download-exit-code` (optional, integer) 84 | 85 | Default: `2` 86 | 87 | Exit code of the plugin if the call to `buildkite-agent artifact download` fails. 88 | 89 | ### `min-tests` (optional, integer) 90 | 91 | Minimum amount of run tests that need to be analyzed or a failure will be reported. It is useful to ensure that tests are actually run and report files to analyze do contain information. 92 | 93 | ### `report-skipped` (optional, boolean) 94 | 95 | Default: `false` 96 | 97 | Will add a list of skipped tests at the end of the annotation. Note that even if there are skipped tests, the annotation may not be added unless other options or results of the processing forces it to. 98 | 99 | ### `report-slowest` (optional) 100 | 101 | Default: `0` 102 | 103 | Include the specified number of slowest tests in the annotation. The annotation will always be shown. 104 | 105 | ### `ruby-image` (optional) 106 | 107 | The docker image to use for running the analysis code. Must be a valid image reference that can run the corresponding ruby code and the agent running the step must be able to pull it if not already present. 108 | 109 | Default: `ruby:3.1-alpine@sha256:a39e26d0598837f08c75a42c8b0886d9ed5cc862c4b535662922ee1d05272fca` 110 | 111 | ### `run-in-docker` (optional, boolean) 112 | 113 | Default: `true` 114 | 115 | Controls whether the JUnit processing should run inside a Docker container. When set to `false`, the processing will run directly on the host using the system's Ruby installation. 116 | 117 | ## Compatibility 118 | 119 | | Elastic Stack | Agent Stack K8s | Hosted (Mac) | Hosted (Linux) | Notes | 120 | | :-----------: | :-------------: | :----: | :----: |:---- | 121 | | ✅ | ⚠️ | ⚠️ | ✅ | **K8s** - Out of the box, requires `run-in-docker: false` and a container image with `ruby` installed
Likely requires some complex podSpec (pending investigation)
**Hosted (Mac)** - instances do not ship with the Docker daemon, but can use a `ruby` binary on the agent | 122 | 123 | - ✅ Fully supported (all combinations of attributes have been tested to pass) 124 | - ⚠️ Partially supported (some combinations cause errors/issues) 125 | - ❌ Not supported 126 | 127 | 128 | ## Developing 129 | 130 | To run testing, shellchecks and plugin linting use use `bk run` with the [Buildkite CLI](https://github.com/buildkite/cli). 131 | 132 | ```bash 133 | bk run 134 | ``` 135 | 136 | Or if you want to run just the plugin tests, you can use the docker [Plugin Tester](https://github.com/buildkite-plugins/buildkite-plugin-tester): 137 | 138 | ```bash 139 | docker run --rm -ti -v "${PWD}":/plugin buildkite/plugin-tester:latest 140 | ``` 141 | 142 | To test the Ruby code with `rake` in docker: 143 | 144 | ```bash 145 | docker-compose run --rm ruby 146 | ``` 147 | 148 | To test your plugin in your builds prior to opening a pull request, you can refer to your fork and SHA from a branch in your `pipeline.yml`. 149 | 150 | ``` 151 | steps: 152 | - label: Annotate 153 | plugins: 154 | - YourGithubHandle/junit-annotate#v2.7.0: 155 | ... 156 | ``` 157 | 158 | ## License 159 | 160 | MIT (see [LICENSE](LICENSE)) 161 | -------------------------------------------------------------------------------- /tests/command.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "${BATS_PLUGIN_PATH}/load.bash" 4 | 5 | # Uncomment to get debug output from each stub 6 | # export MKTEMP_STUB_DEBUG=/dev/tty 7 | # export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty 8 | # export DOCKER_STUB_DEBUG=/dev/tty 9 | # export DU_STUB_DEBUG=/dev/tty 10 | 11 | export artifacts_tmp="tests/tmp/junit-artifacts" 12 | export annotation_tmp="tests/tmp/junit-annotation" 13 | export annotation_input="tests/tmp/annotation.input" 14 | 15 | DOCKER_STUB_DEFAULT_OPTIONS='--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* --env \* \*' 16 | 17 | @test "runs the annotator and creates the annotation" { 18 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 19 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR=false 20 | 21 | stub mktemp \ 22 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 23 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 24 | 25 | stub buildkite-agent \ 26 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 27 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 28 | 29 | stub docker \ 30 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" 31 | 32 | run "$PWD/hooks/command" 33 | 34 | assert_success 35 | 36 | assert_output --partial "Annotation added with context junit and style error" 37 | assert_equal "$(cat "${annotation_input}")" '
Failure
' 38 | 39 | unstub mktemp 40 | unstub buildkite-agent 41 | unstub docker 42 | rm "${annotation_input}" 43 | } 44 | 45 | @test "can define a special context" { 46 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 47 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_CONTEXT="junit_custom_context" 48 | 49 | stub mktemp \ 50 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 51 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 52 | 53 | stub buildkite-agent \ 54 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 55 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 56 | 57 | stub docker \ 58 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" 59 | 60 | run "$PWD/hooks/command" 61 | 62 | assert_success 63 | 64 | assert_output --partial "Annotation added with context junit_custom_context" 65 | 66 | unstub mktemp 67 | unstub buildkite-agent 68 | unstub docker 69 | rm "${annotation_input}" 70 | } 71 | 72 | @test "can pass through optional job uuid file pattern" { 73 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 74 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN="custom_(*)_pattern.xml" 75 | 76 | stub mktemp \ 77 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 78 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 79 | 80 | stub buildkite-agent \ 81 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 82 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 83 | 84 | stub docker \ 85 | "--log-level error run --rm --volume \* --volume \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN='custom_(*)_pattern.xml' --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" 86 | 87 | run "$PWD/hooks/command" 88 | 89 | assert_success 90 | 91 | assert_output --partial "Annotation added" 92 | 93 | unstub mktemp 94 | unstub buildkite-agent 95 | unstub docker 96 | rm "${annotation_input}" 97 | } 98 | 99 | @test "can pass through optional failure format" { 100 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 101 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT="file" 102 | 103 | stub mktemp \ 104 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 105 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 106 | 107 | stub buildkite-agent \ 108 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 109 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 110 | 111 | stub docker \ 112 | "--log-level error run --rm --volume \* --volume \* --env \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT='file' --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" 113 | 114 | run "$PWD/hooks/command" 115 | 116 | assert_success 117 | 118 | assert_output --partial "Annotation added" 119 | 120 | unstub mktemp 121 | unstub buildkite-agent 122 | unstub docker 123 | rm "${annotation_input}" 124 | } 125 | 126 | @test "doesn't create annotation unless there's failures" { 127 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 128 | 129 | stub mktemp \ 130 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 131 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 132 | 133 | stub buildkite-agent \ 134 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" 135 | 136 | stub docker \ 137 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : echo 'Total tests: 0'" 138 | 139 | run "$PWD/hooks/command" 140 | 141 | assert_success 142 | 143 | unstub mktemp 144 | unstub buildkite-agent 145 | unstub docker 146 | } 147 | 148 | @test "creates annotation with no failures but always annotate" { 149 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 150 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ALWAYS_ANNOTATE=1 151 | 152 | stub mktemp \ 153 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 154 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 155 | 156 | stub buildkite-agent \ 157 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 158 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 159 | 160 | stub docker \ 161 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : echo 'Total tests: 0'" 162 | 163 | run "$PWD/hooks/command" 164 | 165 | assert_success 166 | assert_output --partial "Total tests: 0" 167 | assert_output --partial "Will create annotation anyways" 168 | assert_equal "$(cat "${annotation_input}")" 'Total tests: 0' 169 | 170 | unstub mktemp 171 | unstub buildkite-agent 172 | unstub docker 173 | } 174 | 175 | @test "errors without the 'artifacts' property set" { 176 | run "$PWD/hooks/command" 177 | 178 | assert_failure 179 | 180 | assert_output --partial "Missing artifacts configuration for the plugin" 181 | refute_output --partial ":junit:" 182 | } 183 | 184 | @test "fails if the annotation is larger than 1MB even after summary" { 185 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 186 | 187 | stub mktemp \ 188 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 189 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 190 | 191 | # 1KB over the 1MB size limit of annotations 192 | stub du \ 193 | "-k \* : echo 1025$'\t'\$2" \ 194 | "-k \* : echo 1025$'\t'\$2" 195 | 196 | stub buildkite-agent \ 197 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" 198 | 199 | stub docker \ 200 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" 201 | 202 | run "$PWD/hooks/command" 203 | 204 | assert_success 205 | 206 | assert_output --partial "Failures too large to annotate" 207 | assert_output --partial "failures are too large to create a build annotation" 208 | 209 | unstub docker 210 | unstub du 211 | unstub buildkite-agent 212 | unstub mktemp 213 | } 214 | 215 | @test "creates summary annotation if original is larger than 1MB" { 216 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 217 | 218 | stub mktemp \ 219 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 220 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 221 | 222 | # 1KB over the 1MB size limit of annotations 223 | stub du \ 224 | "-k \* : echo 1025$'\t'\$2" \ 225 | "-k \* : echo 10$'\t'\$2" 226 | 227 | stub buildkite-agent \ 228 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 229 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 230 | 231 | stub docker \ 232 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" 233 | 234 | run "$PWD/hooks/command" 235 | 236 | assert_success 237 | 238 | assert_output --partial "Failures too large to annotate" 239 | assert_output --partial "using a simplified annotation" 240 | assert_equal "6 ${annotation_input}" "$(wc -l "${annotation_input}" | cut -f 1)" 241 | 242 | unstub docker 243 | unstub du 244 | unstub buildkite-agent 245 | unstub mktemp 246 | rm "${annotation_input}" 247 | } 248 | 249 | @test "returns an error if fail-build-on-error is true" { 250 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 251 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR=true 252 | 253 | stub mktemp \ 254 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 255 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 256 | 257 | stub buildkite-agent \ 258 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 259 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 260 | 261 | stub docker \ 262 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" 263 | 264 | run "$PWD/hooks/command" 265 | 266 | assert_failure 267 | 268 | unstub mktemp 269 | unstub buildkite-agent 270 | unstub docker 271 | rm "${annotation_input}" 272 | } 273 | 274 | @test "returns an error if fail-build-on-error is true and annotation is too large" { 275 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 276 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR=true 277 | 278 | stub mktemp \ 279 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 280 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 281 | 282 | # 1KB over the 1MB size limit of annotations 283 | stub du \ 284 | "-k \* : echo 1025$'\t'\$2" \ 285 | "-k \* : echo 1025$'\t'\$2" 286 | 287 | stub buildkite-agent \ 288 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" 289 | 290 | stub docker \ 291 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" 292 | 293 | run "$PWD/hooks/command" 294 | 295 | assert_failure 296 | 297 | assert_output --partial "Failures too large to annotate" 298 | 299 | unstub mktemp 300 | unstub du 301 | unstub buildkite-agent 302 | unstub docker 303 | } 304 | 305 | @test "error bubbles up when ruby code fails with anything but 64" { 306 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 307 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR=false 308 | 309 | stub mktemp \ 310 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 311 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 312 | 313 | stub buildkite-agent \ 314 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" 315 | 316 | stub docker \ 317 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 147" 318 | 319 | run "$PWD/hooks/command" 320 | 321 | assert_failure 147 322 | 323 | assert_output --partial "Error when processing JUnit tests" 324 | 325 | unstub mktemp 326 | unstub buildkite-agent 327 | unstub docker 328 | } 329 | 330 | @test "error bubbles up when agent download fails" { 331 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 332 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR=false 333 | 334 | stub mktemp \ 335 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 336 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 337 | 338 | stub buildkite-agent \ 339 | "artifact download \* \* : exit 1" 340 | 341 | run "$PWD/hooks/command" 342 | 343 | assert_failure 2 344 | 345 | assert_output --partial "Could not download artifacts" 346 | 347 | unstub mktemp 348 | unstub buildkite-agent 349 | } 350 | 351 | @test "customize error when agent download fails" { 352 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 353 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILED_DOWNLOAD_EXIT_CODE=5 354 | 355 | stub mktemp \ 356 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 357 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 358 | 359 | stub buildkite-agent \ 360 | "artifact download \* \* : exit 1" 361 | 362 | run "$PWD/hooks/command" 363 | 364 | assert_failure 5 365 | 366 | assert_output --partial "Could not download artifacts" 367 | 368 | unstub mktemp 369 | unstub buildkite-agent 370 | } 371 | 372 | @test "creates annotation with no failures but min tests triggers" { 373 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 374 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS=1 375 | 376 | stub mktemp \ 377 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 378 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 379 | 380 | stub buildkite-agent \ 381 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 382 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 383 | 384 | stub docker \ 385 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : echo 'Total tests: 0'" 386 | 387 | run "$PWD/hooks/command" 388 | 389 | assert_failure 390 | assert_output --partial "Total tests: 0" 391 | assert_output --partial "Less than 1 tests analyzed" 392 | assert_equal "$(cat "${annotation_input}")" 'Total tests: 0' 393 | 394 | unstub mktemp 395 | unstub buildkite-agent 396 | unstub docker 397 | } 398 | 399 | @test "no failures and min-tests ok does not create annotation" { 400 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 401 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS=12 402 | 403 | stub mktemp \ 404 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 405 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 406 | 407 | stub buildkite-agent \ 408 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" 409 | 410 | stub docker \ 411 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : echo 'Total tests: 100'" 412 | 413 | run "$PWD/hooks/command" 414 | 415 | assert_success 416 | assert_output --partial "Total tests: 100" 417 | refute_output --partial "Less than 12 tests analyzed" 418 | 419 | unstub mktemp 420 | unstub buildkite-agent 421 | unstub docker 422 | } 423 | 424 | @test "min-tests doesn't interfere with actual failures" { 425 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 426 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS=10000 427 | 428 | stub mktemp \ 429 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 430 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 431 | 432 | stub buildkite-agent \ 433 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 434 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 435 | 436 | stub docker \ 437 | "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" 438 | 439 | run "$PWD/hooks/command" 440 | 441 | assert_success 442 | assert_output --partial "Total tests: 2" 443 | 444 | unstub mktemp 445 | unstub buildkite-agent 446 | unstub docker 447 | } 448 | 449 | @test "runs the annotator and creates the annotation with special image" { 450 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 451 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_RUBY_IMAGE="ruby:special" 452 | 453 | stub mktemp \ 454 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 455 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 456 | 457 | stub buildkite-agent \ 458 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 459 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 460 | 461 | stub docker \ 462 | "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* --env \* ruby:special ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" 463 | 464 | run "$PWD/hooks/command" 465 | 466 | assert_success 467 | 468 | assert_output --partial "Annotation added with context junit and style error" 469 | assert_equal "$(cat "${annotation_input}")" '
Failure
' 470 | 471 | unstub mktemp 472 | unstub buildkite-agent 473 | unstub docker 474 | rm "${annotation_input}" 475 | } 476 | 477 | @test "can customize skipped tests variable" { 478 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 479 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED="true" 480 | 481 | stub mktemp \ 482 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 483 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 484 | 485 | stub buildkite-agent \ 486 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 487 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 488 | 489 | stub docker \ 490 | "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED='true' \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" 491 | 492 | run "$PWD/hooks/command" 493 | 494 | assert_success 495 | 496 | assert_output --partial "Annotation added" 497 | 498 | unstub mktemp 499 | unstub buildkite-agent 500 | unstub docker 501 | rm "${annotation_input}" 502 | } 503 | 504 | @test "creates annotation with no failures but with slowest tests trigger" { 505 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 506 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST=5 507 | 508 | stub mktemp \ 509 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 510 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 511 | 512 | stub buildkite-agent \ 513 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 514 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 515 | 516 | stub docker \ 517 | "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST=5 --env \* \* ruby /src/bin/annotate /junits : cat tests/2-slowest-tests.output" 518 | 519 | run "$PWD/hooks/command" 520 | 521 | assert_success 522 | 523 | assert_output --partial "Create annotation with slowest tests" 524 | assert_output --partial "5 slowest tests" 525 | 526 | unstub mktemp 527 | unstub buildkite-agent 528 | unstub docker 529 | rm "${annotation_input}" 530 | } 531 | 532 | @test "does not run in docker when run-in-docker is false" { 533 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" 534 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR=false 535 | export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_RUN_IN_DOCKER=false 536 | 537 | stub mktemp \ 538 | "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ 539 | "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" 540 | 541 | stub buildkite-agent \ 542 | "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ 543 | "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" 544 | 545 | stub ruby \ 546 | "/plugin/hooks/../ruby/bin/annotate /plugin/${artifacts_tmp} : echo '
Failure
' && exit 64" 547 | 548 | run "$PWD/hooks/command" 549 | 550 | assert_success 551 | 552 | assert_output --partial "Annotation added with context junit and style error" 553 | assert_equal "$(cat "${annotation_input}")" '
Failure
' 554 | 555 | unstub mktemp 556 | unstub buildkite-agent 557 | unstub ruby 558 | rm "${annotation_input}" 559 | } -------------------------------------------------------------------------------- /ruby/tests/annotate_test.rb: -------------------------------------------------------------------------------- 1 | require 'minitest/autorun' 2 | require 'open3' 3 | 4 | describe "Junit annotate plugin parser" do 5 | it "handles no failures" do 6 | stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/no-test-failures/") 7 | 8 | assert_equal stderr, <<~OUTPUT 9 | Parsing junit-1.xml 10 | Parsing junit-2.xml 11 | Parsing junit-3.xml 12 | --- ✍️ Preparing annotation 13 | OUTPUT 14 | 15 | assert_equal stdout, <<~OUTPUT 16 | Failures: 0 17 | Errors: 0 18 | Skipped: 0 19 | Total tests: 8 20 | OUTPUT 21 | 22 | assert_equal 0, status.exitstatus 23 | end 24 | 25 | it "handles failures across multiple files" do 26 | stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/two-test-failures/") 27 | 28 | assert_equal stderr, <<~OUTPUT 29 | Parsing junit-1.xml 30 | Parsing junit-2.xml 31 | Parsing junit-3.xml 32 | --- ✍️ Preparing annotation 33 | OUTPUT 34 | 35 | assert_equal stdout, <<~OUTPUT 36 | Failures: 4 37 | Errors: 0 38 | Skipped: 0 39 | Total tests: 6 40 | 41 |
42 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec 43 | 44 |

expected: 250 got: 500 (compared using eql?)

45 | 46 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
 47 | 
 48 |         expected: 250
 49 |              got: 500
 50 | 
 51 |         (compared using eql?)
 52 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
 53 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
 54 |       ./spec/support/log.rb:17:in `run'
 55 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
56 | 57 | in Job #1 58 |
59 | 60 |
61 | Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec 62 | 63 |

expected: 700 got: 500 (compared using eql?)

64 | 65 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
 66 | 
 67 |         expected: 700
 68 |              got: 500
 69 | 
 70 |         (compared using eql?)
 71 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
 72 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
 73 |       ./spec/support/log.rb:17:in `run'
 74 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
75 | 76 | in Job #2 77 |
78 | 79 |
80 | Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec 81 | 82 |

expected: 700 got: 500 (compared using eql?)

83 | 84 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
 85 | 
 86 |         expected: 700
 87 |              got: 500
 88 | 
 89 |         (compared using eql?)
 90 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
 91 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
 92 |       ./spec/support/log.rb:17:in `run'
 93 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
94 | 95 | in Job #3 96 |
97 | 98 |
99 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec 100 | 101 |

expected: 250 got: 500 (compared using eql?)

102 | 103 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
104 | 
105 |         expected: 250
106 |              got: 500
107 | 
108 |         (compared using eql?)
109 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
110 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
111 |       ./spec/support/log.rb:17:in `run'
112 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
113 | 114 | in Job #3 115 |
116 | OUTPUT 117 | 118 | assert_equal 64, status.exitstatus 119 | end 120 | 121 | it "handles failures and errors across multiple files" do 122 | stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/test-failure-and-error/") 123 | 124 | assert_equal stderr, <<~OUTPUT 125 | Parsing junit-1.xml 126 | Parsing junit-2.xml 127 | Parsing junit-3.xml 128 | --- ✍️ Preparing annotation 129 | OUTPUT 130 | 131 | assert_equal stdout, <<~OUTPUT 132 | Failures: 2 133 | Errors: 2 134 | Skipped: 0 135 | Total tests: 6 136 | 137 |
138 | Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec 139 | 140 |

expected: 700 got: 500 (compared using eql?)

141 | 142 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
143 | 
144 |         expected: 700
145 |              got: 500
146 | 
147 |         (compared using eql?)
148 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
149 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
150 |       ./spec/support/log.rb:17:in `run'
151 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
152 | 153 | in Job #2 154 |
155 | 156 |
157 | Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec 158 | 159 |

expected: 700 got: 500 (compared using eql?)

160 | 161 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
162 | 
163 |         expected: 700
164 |              got: 500
165 | 
166 |         (compared using eql?)
167 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
168 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
169 |       ./spec/support/log.rb:17:in `run'
170 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
171 | 172 | in Job #3 173 |
174 | 175 |
176 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec 177 | 178 |

expected: 250 got: 500 (compared using eql?)

179 | 180 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
181 | 
182 |         expected: 250
183 |              got: 500
184 | 
185 |         (compared using eql?)
186 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
187 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
188 |       ./spec/support/log.rb:17:in `run'
189 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
190 | 191 | in Job #1 192 |
193 | 194 |
195 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec 196 | 197 |

expected: 250 got: 500 (compared using eql?)

198 | 199 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
200 | 
201 |         expected: 250
202 |              got: 500
203 | 
204 |         (compared using eql?)
205 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
206 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
207 |       ./spec/support/log.rb:17:in `run'
208 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
209 | 210 | in Job #3 211 |
212 | OUTPUT 213 | 214 | assert_equal 64, status.exitstatus 215 | end 216 | 217 | it "accepts custom regex filename patterns for job id" do 218 | stdout, stderr, status = Open3.capture3("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN=junit-(.*)-custom-pattern.xml", "#{__dir__}/../bin/annotate", "#{__dir__}/custom-job-uuid-pattern/") 219 | 220 | assert_equal stderr, <<~OUTPUT 221 | Parsing junit-123-456-custom-pattern.xml 222 | --- ✍️ Preparing annotation 223 | OUTPUT 224 | 225 | assert_equal stdout, <<~OUTPUT 226 | Failures: 1 227 | Errors: 0 228 | Skipped: 0 229 | Total tests: 2 230 | 231 |
232 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec 233 | 234 |

expected: 250 got: 500 (compared using eql?)

235 | 236 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
237 | 
238 |         expected: 250
239 |              got: 500
240 | 
241 |         (compared using eql?)
242 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
243 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
244 |       ./spec/support/log.rb:17:in `run'
245 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
246 | 247 | in Job #123-456 248 |
249 | OUTPUT 250 | 251 | assert_equal 64, status.exitstatus 252 | end 253 | 254 | it "uses the file path instead of classname for annotation content when specified" do 255 | stdout, stderr, status = Open3.capture3("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT=file", "#{__dir__}/../bin/annotate", "#{__dir__}/test-failure-and-error/") 256 | 257 | assert_equal stderr, <<~OUTPUT 258 | Parsing junit-1.xml 259 | Parsing junit-2.xml 260 | Parsing junit-3.xml 261 | --- ✍️ Preparing annotation 262 | OUTPUT 263 | 264 | assert_equal stdout, <<~OUTPUT 265 | Failures: 2 266 | Errors: 2 267 | Skipped: 0 268 | Total tests: 6 269 | 270 |
271 | Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in ./spec/models/account_spec.rb 272 | 273 |

expected: 700 got: 500 (compared using eql?)

274 | 275 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
276 | 
277 |         expected: 700
278 |              got: 500
279 | 
280 |         (compared using eql?)
281 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
282 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
283 |       ./spec/support/log.rb:17:in `run'
284 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
285 | 286 | in Job #2 287 |
288 | 289 |
290 | Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in ./spec/models/account_spec.rb 291 | 292 |

expected: 700 got: 500 (compared using eql?)

293 | 294 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
295 | 
296 |         expected: 700
297 |              got: 500
298 | 
299 |         (compared using eql?)
300 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
301 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
302 |       ./spec/support/log.rb:17:in `run'
303 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
304 | 305 | in Job #3 306 |
307 | 308 |
309 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in ./spec/models/account_spec.rb 310 | 311 |

expected: 250 got: 500 (compared using eql?)

312 | 313 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
314 | 
315 |         expected: 250
316 |              got: 500
317 | 
318 |         (compared using eql?)
319 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
320 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
321 |       ./spec/support/log.rb:17:in `run'
322 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
323 | 324 | in Job #1 325 |
326 | 327 |
328 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in ./spec/models/account_spec.rb 329 | 330 |

expected: 250 got: 500 (compared using eql?)

331 | 332 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
333 | 
334 |         expected: 250
335 |              got: 500
336 | 
337 |         (compared using eql?)
338 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
339 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
340 |       ./spec/support/log.rb:17:in `run'
341 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
342 | 343 | in Job #3 344 |
345 | OUTPUT 346 | 347 | assert_equal 64, status.exitstatus 348 | end 349 | 350 | it "handles failures across multiple files in sub dirs" do 351 | stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/tests-in-sub-dirs/") 352 | 353 | assert_equal stderr, <<~OUTPUT 354 | Parsing sub-dir/junit-1.xml 355 | Parsing sub-dir/junit-2.xml 356 | Parsing sub-dir/junit-3.xml 357 | --- ✍️ Preparing annotation 358 | OUTPUT 359 | 360 | assert_equal stdout, <<~OUTPUT 361 | Failures: 4 362 | Errors: 0 363 | Skipped: 0 364 | Total tests: 6 365 | 366 |
367 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec 368 | 369 |

expected: 250 got: 500 (compared using eql?)

370 | 371 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
372 | 
373 |         expected: 250
374 |              got: 500
375 | 
376 |         (compared using eql?)
377 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
378 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
379 |       ./spec/support/log.rb:17:in `run'
380 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
381 | 382 | in Job #1 383 |
384 | 385 |
386 | Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec 387 | 388 |

expected: 700 got: 500 (compared using eql?)

389 | 390 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
391 | 
392 |         expected: 700
393 |              got: 500
394 | 
395 |         (compared using eql?)
396 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
397 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
398 |       ./spec/support/log.rb:17:in `run'
399 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
400 | 401 | in Job #2 402 |
403 | 404 |
405 | Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec 406 | 407 |

expected: 700 got: 500 (compared using eql?)

408 | 409 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
410 | 
411 |         expected: 700
412 |              got: 500
413 | 
414 |         (compared using eql?)
415 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
416 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
417 |       ./spec/support/log.rb:17:in `run'
418 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
419 | 420 | in Job #3 421 |
422 | 423 |
424 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec 425 | 426 |

expected: 250 got: 500 (compared using eql?)

427 | 428 | 
Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
429 | 
430 |         expected: 250
431 |              got: 500
432 | 
433 |         (compared using eql?)
434 |       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
435 |       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
436 |       ./spec/support/log.rb:17:in `run'
437 |       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
438 | 439 | in Job #3 440 |
441 | OUTPUT 442 | 443 | assert_equal 64, status.exitstatus 444 | end 445 | 446 | it "handles empty failure bodies" do 447 | stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/empty-failure-body/") 448 | 449 | assert_equal stderr, <<~OUTPUT 450 | Parsing junit.xml 451 | --- ✍️ Preparing annotation 452 | OUTPUT 453 | 454 | assert_equal stdout, <<~OUTPUT 455 | Failures: 1 456 | Errors: 0 457 | Skipped: 0 458 | Total tests: 2 459 | 460 |
461 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec 462 | 463 |

expected: 250 got: 500 (compared using eql?)

464 | 465 |
466 | OUTPUT 467 | 468 | assert_equal 64, status.exitstatus 469 | end 470 | 471 | it "handles miss message attributes" do 472 | stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/missing-message-attribute/") 473 | 474 | assert_equal stderr, <<~OUTPUT 475 | Parsing junit.xml 476 | --- ✍️ Preparing annotation 477 | OUTPUT 478 | 479 | assert_equal stdout, <<~OUTPUT 480 | Failures: 1 481 | Errors: 2 482 | Skipped: 0 483 | Total tests: 4 484 | 485 |
486 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec 487 | 488 |
489 | 490 |
491 | Account#maximum_jobs_added_by_pipeline_changer returns 100 by default in spec.models.account_spec 492 | 493 |
494 | 495 |
496 | Account#maximum_jobs_added_by_pipeline_changer returns 50 by default in spec.models.account_spec 497 | 498 |
499 | OUTPUT 500 | 501 | assert_equal 64, status.exitstatus 502 | end 503 | 504 | it "handles cdata formatted XML files" do 505 | stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/failure-with-cdata/") 506 | 507 | assert_equal stderr, <<~OUTPUT 508 | Parsing junit.xml 509 | --- ✍️ Preparing annotation 510 | OUTPUT 511 | 512 | assert_equal stdout, <<~OUTPUT 513 | Failures: 0 514 | Errors: 1 515 | Skipped: 0 516 | Total tests: 2 517 | 518 |
519 | Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec 520 | 521 |

expected: 250 got: 500 (compared using eql?)

522 | 523 | 
First line of failure output
524 |             Second line of failure output
525 | 526 |
527 | OUTPUT 528 | 529 | assert_equal 64, status.exitstatus 530 | end 531 | 532 | it "reports specified amount of slowest tests" do 533 | stdout, stderr, status = Open3.capture3("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST=5", "#{__dir__}/../bin/annotate", "#{__dir__}/no-test-failures/") 534 | 535 | assert_equal stderr, <<~OUTPUT 536 | Parsing junit-1.xml 537 | Parsing junit-2.xml 538 | Parsing junit-3.xml 539 | --- ✍️ Preparing annotation 540 | Reporting slowest tests ⏱ 541 | OUTPUT 542 | 543 | assert_equal stdout, <<~OUTPUT 544 | Failures: 0 545 | Errors: 0 546 | Skipped: 0 547 | Total tests: 8 548 | 549 |
550 | 5 slowest tests 551 | 552 | 553 | 554 | 555 | 556 | 557 | 558 | 559 | 560 | 561 |
UnitTestTime
spec.models.account_specAccount#maximum_jobs_added_by_pipeline_changer returns 250 by default0.977127
spec.models.account_specAccount#maximum_jobs_added_by_pipeline_changer returns 250 by default0.967127
spec.models.account_specAccount#maximum_jobs_added_by_pipeline_changer returns 500 if the account is ABC0.620013
spec.models.account_specAccount#maximum_jobs_added_by_pipeline_changer returns 900 if the account is F000.520013
spec.models.account_specAccount#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ0.420013
562 |
563 | OUTPUT 564 | 565 | assert_equal 0, status.exitstatus 566 | end 567 | 568 | it "handles junit dir paths with hidden directories" do 569 | stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/.tests-in-hidden-dir/") 570 | 571 | assert_equal stderr, <<~OUTPUT 572 | Parsing junit-1.xml 573 | --- ✍️ Preparing annotation 574 | OUTPUT 575 | 576 | assert_equal stdout, <<~OUTPUT 577 | Failures: 0 578 | Errors: 0 579 | Skipped: 0 580 | Total tests: 2 581 | OUTPUT 582 | 583 | assert_equal 0, status.exitstatus 584 | end 585 | 586 | it "correctly parses skipped tests" do 587 | stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/skipped-test/") 588 | 589 | assert_equal stderr, <<~OUTPUT 590 | Parsing junit.xml 591 | --- ✍️ Preparing annotation 592 | OUTPUT 593 | 594 | assert_equal stdout, <<~OUTPUT 595 | Failures: 0 596 | Errors: 0 597 | Skipped: 1 598 | Total tests: 2 599 | OUTPUT 600 | 601 | assert_equal 0, status.exitstatus 602 | end 603 | 604 | it "can report skipped tests" do 605 | stdout, stderr, status = Open3.capture3("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED=true", "#{__dir__}/../bin/annotate", "#{__dir__}/skipped-test/") 606 | 607 | assert_equal stderr, <<~OUTPUT 608 | Parsing junit.xml 609 | --- ✍️ Preparing annotation 610 | Reporting skipped tests 611 | OUTPUT 612 | 613 | assert_equal stdout, <<~OUTPUT 614 | Failures: 0 615 | Errors: 0 616 | Skipped: 1 617 | Total tests: 2 618 | 619 |
620 | 1 tests skipped 621 | 622 |
    623 |
  1. Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec (because of reasons)
    2. 624 |
625 |
626 | OUTPUT 627 | 628 | assert_equal 0, status.exitstatus 629 | end 630 | end 631 | --------------------------------------------------------------------------------