├── .buildkite ├── hooks │ └── pre-command ├── pipeline.sh └── pipeline.yml ├── .dockerignore ├── .gitignore ├── .rspec ├── .rubocop.yml ├── .ruby-version ├── CHANGELOG.md ├── Dockerfile ├── Gemfile ├── Gemfile.lock ├── LICENSE.txt ├── README.md ├── RELEASE.md ├── Rakefile ├── bin ├── console ├── run ├── run-dev └── setup ├── doc ├── details.png └── summary.png ├── docker-compose.yml ├── hooks └── command ├── lib ├── test_summary_buildkite_plugin.rb └── test_summary_buildkite_plugin │ ├── agent.rb │ ├── failure.rb │ ├── formatter.rb │ ├── haml_render.rb │ ├── input.rb │ ├── runner.rb │ ├── tap.rb │ ├── truncater.rb │ ├── utils.rb │ └── version.rb ├── plugin.yml ├── spec ├── sample_artifacts │ ├── checkstyle.xml │ ├── eslint-00112233-0011-0011-0011-001122334455.txt │ ├── example.tap │ ├── rspec-01234567-0011-0011-0011-001122334455.xml │ ├── rspec-76543210-0011-0011-0011-001122334455.xml │ ├── rspec-aabbccdd-0011-0011-0011-001122334455.xml │ ├── rubocop.txt │ ├── version_12.tap │ ├── xunit-no-file.xml │ └── xunit.xml ├── spec_helper.rb ├── support │ └── stubs.rb ├── test_summary_buildkite_plugin │ ├── failure_spec.rb │ ├── formatter_spec.rb │ ├── input_spec.rb │ ├── runner_spec.rb │ ├── tap_spec.rb │ └── truncater_spec.rb └── test_summary_buildkite_plugin_spec.rb └── templates ├── details └── failures.html.haml ├── summary ├── failures.html.haml └── footer.html.haml ├── test_layout.html.haml └── truncater_exception.html.haml /.buildkite/hooks/pre-command: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | 4 | # Make a world-writable directory for managing artifacts inside and outside a container 5 | echo 'Creating artifacts directory' 6 | mkdir -p artifacts 7 | chmod 0777 artifacts 8 | -------------------------------------------------------------------------------- /.buildkite/pipeline.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | HASH=$(git rev-parse HEAD) 7 | sed "s/@VERSION@/$HASH/g" "$SCRIPT_DIR/pipeline.yml" 8 | -------------------------------------------------------------------------------- /.buildkite/pipeline.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - label: ":rspec: rspec" 3 | command: bundle exec rspec --format progress --format RspecJunitFormatter --out artifacts/rspec.xml 4 | timeout: 5 5 | artifact_paths: "artifacts/*" 6 | plugins: 7 | - docker-compose#v2.4.0: 8 | run: test 9 | 10 | - label: ":rubocop: rubocop" 11 | command: bundle exec rubocop --format progress --format emacs --out artifacts/rubocop.txt 12 | timeout: 5 13 | artifact_paths: "artifacts/*" 14 | plugins: 15 | - docker-compose#v2.4.0: 16 | run: test 17 | 18 | - label: ":sparkles: plugin lint" 19 | plugins: 20 | - plugin-linter#v2.0.0: 21 | id: bugcrowd/test-summary 22 | 23 | - label: ":arrow_up: artifacts" 24 | command: buildkite-agent artifact upload spec/sample_artifacts/**/* 25 | 26 | - wait: ~ 27 | continue_on_failure: true 28 | 29 | - label: ":pencil: annotate" 30 | plugins: 31 | - bugcrowd/test-summary#@VERSION@: 32 | inputs: 33 | - label: ":rspec: rspec" 34 | artifact_path: artifacts/rspec.xml 35 | type: junit 36 | strip_colors: true 37 | - label: ":rubocop: rubocop" 38 | artifact_path: artifacts/rubocop.txt 39 | type: oneline 40 | formatter: 41 | type: details 42 | context: failures 43 | 44 | - label: ":pencil: summary" 45 | plugins: 46 | - bugcrowd/test-summary#@VERSION@: 47 | inputs: 48 | - label: ":rspec: rspec" 49 | artifact_path: spec/sample_artifacts/rspec* 50 | type: junit 51 | encoding: UTF-8 52 | strip_colors: true 53 | - label: ":female-technologist: xunit" 54 | artifact_path: spec/sample_artifacts/xunit*.xml 55 | type: junit 56 | - label: ":camel: tap" 57 | artifact_path: spec/sample_artifacts/example.tap 58 | type: tap 59 | - label: ":camel: tap v12" 60 | artifact_path: spec/sample_artifacts/version_12.tap 61 | type: tap 62 | - label: ":eslint: eslint" 63 | artifact_path: spec/sample_artifacts/eslint-*.txt 64 | type: oneline 65 | # ignore the last two lines 66 | crop: 67 | start: 0 68 | end: 2 69 | - label: ":rubocop: rubocop" 70 | artifact_path: spec/sample_artifacts/rubocop.txt 71 | type: oneline 72 | - label: ":greencheck: checkstyle" 73 | artifact_path: spec/sample_artifacts/checkstyle.xml 74 | type: checkstyle 75 | formatter: 76 | type: summary 77 | show_first: 3 78 | context: simple-annotation 79 | style: default 80 | fail_on_error: true 81 | 82 | # Add test summary, including failure details 83 | - label: ":pencil: details" 84 | plugins: 85 | - bugcrowd/test-summary#@VERSION@: 86 | inputs: 87 | - label: ":rspec: rspec" 88 | artifact_path: spec/sample_artifacts/rspec* 89 | type: junit 90 | encoding: UTF-8 91 | strip_colors: true 92 | message: false 93 | summary: 94 | format: '%{location}: %{testcase.name}' 95 | details_regex: '(?\S+:\d+)[^\n]*\z' 96 | - label: ":female-technologist: xunit" 97 | artifact_path: spec/sample_artifacts/xunit*.xml 98 | type: junit 99 | - label: ":camel: tap" 100 | artifact_path: spec/sample_artifacts/example.tap 101 | type: tap 102 | - label: ":camel: tap v12" 103 | artifact_path: spec/sample_artifacts/version_12.tap 104 | type: tap 105 | - label: ":eslint: eslint" 106 | artifact_path: spec/sample_artifacts/eslint-*.txt 107 | type: oneline 108 | # ignore the last two lines 109 | crop: 110 | start: 0 111 | end: 2 112 | - label: ":rubocop: rubocop" 113 | artifact_path: spec/sample_artifacts/rubocop.txt 114 | type: oneline 115 | - label: ":greencheck: checkstyle" 116 | artifact_path: spec/sample_artifacts/checkstyle.xml 117 | type: checkstyle 118 | - label: "missing" 119 | artifact_path: spec/sample_artifacts/does_not_exist 120 | type: oneline 121 | formatter: 122 | type: details 123 | show_first: 3 124 | context: verbose-annotation 125 | style: info 126 | fail_on_error: false 127 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | tmp 3 | vendor/bundle 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /pkg/ 6 | /spec/reports/ 7 | /tmp/ 8 | 9 | # rspec failure tracking 10 | .rspec_status 11 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper 4 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | Documentation: 2 | Enabled: false 3 | 4 | Metrics/AbcSize: 5 | Max: 25 6 | 7 | Metrics/BlockLength: 8 | ExcludedMethods: 9 | - context 10 | - define 11 | - describe 12 | - it 13 | - let 14 | - let! 15 | - scenario 16 | - shared_context 17 | - shared_examples 18 | - shared_examples_for 19 | 20 | Metrics/CyclomaticComplexity: 21 | Max: 10 22 | 23 | Metrics/LineLength: 24 | Max: 120 25 | 26 | Metrics/MethodLength: 27 | Max: 15 28 | 29 | Metrics/PerceivedComplexity: 30 | Max: 10 31 | 32 | Style/GuardClause: 33 | Enabled: false 34 | 35 | Layout/MultilineMethodCallIndentation: 36 | EnforcedStyle: indented 37 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.5.1 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) 5 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | ## [v1.11.0](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.10.0...v1.11.0) - 2020-01-28 10 | - Forward `HTTP_PROXY` ENV var to respect proxies [#58](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/58) 11 | 12 | ## [v1.10.0](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.9.1...v1.10.0) - 2019-10-12 13 | - Support TAP version 12 plus skipped and TODO tests [#54](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/54) 14 | 15 | ## [v1.9.1](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.9.0...v1.9.1) - 2019-05-30 16 | - Fix Checkstyle formatter to be more tolerant of optional attributes [#52](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/52) 17 | 18 | ## [v1.9.0](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.8.0...v1.9.0) - 2019-02-21 19 | - Fix bin/run-dev to work with the new array syntax [#45](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/45) (thanks @philwo) 20 | - Update buildkite css files in test layout [#46](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/46) 21 | - Update `MAX_MARKDOWN_SIZE` to 100kb [#47](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/47) (thanks @ticky) 22 | - Move docker images to the bugcrowd organisation [#48](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/48) 23 | 24 | ## [v1.8.0](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.7.2...v1.8.0) - 2018-12-14 25 | - Correctly link to jobs post buildkite update [#43](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/43) 26 | - JUnit: Allow disabling the message or details if they aren't relevant [#44](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/44) 27 | 28 | ## [v1.7.2](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.7.1...v1.7.2) - 2018-11-20 29 | - Mount `/tmp` into the container to support `agent-socket` experimental feature [#40](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/40) (thanks @dreyks) 30 | - Update README example to use plugin array syntax [#41](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/41) 31 | 32 | ## [v1.7.1](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.7.0...v1.7.1) - 2018-10-11 33 | - Fix error handling when truncating and an input fails to download [#38](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/38) 34 | 35 | ## [v1.7.0](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.6.0...v1.7.0) - 2018-09-26 36 | - JUnit: Expose classname if file attribute does not exist 37 | [#33](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/33)/[#35](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/35) (thanks @timnew) 38 | - Add support for checkstyle 39 | [#34](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/34)/[#36](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/36) (thanks @timnew) 40 | 41 | ## [v1.6.0](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.5.0...v1.6.0) - 2018-08-06 42 | - Remove undocumented count-only formatter [#28](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/28) 43 | - JUnit: support custom summary formats [#29](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/29) 44 | 45 | ## [v1.5.0](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.4.0...v1.5.0) - 2018-07-25 46 | - JUnit: Don't show an empty message when it's not provided [#21](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/21) 47 | - Remove redcarpet workarounds [#22](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/22) 48 | - Avoid blank lines in html details because CommonMark [#25](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/25) 49 | (thanks for the bug report by @joscha) 50 | 51 | ## [v1.4.0](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.3.0...v1.4.0) - 2018-06-17 52 | - Update plugin.yml and re-enable plugin-linter [#14](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/14) (@toolmantim) 53 | - Workaround redcarpet formatting issues [#17](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/17) 54 | - Truncate failures when markdown is too large [#18](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/18) 55 | 56 | ## [v1.3.0](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.2.0...v1.3.0) - 2018-05-10 57 | - Return zero exit status on error [#11](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/11) 58 | - Junit: Handle testsuite objects nested inside testsuites [#12](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/12) 59 | - Fallback to simpler formats if markdown is too large [#13](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/13) 60 | 61 | ## [v1.2.0](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.1.0...v1.2.0) - 2018-05-04 62 | - HTML escape output [#6](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/6) 63 | - Junit: Support `error` elements and include the message [#7](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/7) 64 | 65 | ## [v1.1.0](https://github.com/bugcrowd/test-summary-buildkite-plugin/compare/v1.0.0...v1.1.0) - 2018-05-01 66 | 67 | - Remove nokogiri dependency and use ruby alpine [#1](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/1) 68 | - plugin.yml [#2](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/2) 69 | - Links to the relevant jobs [#3](https://github.com/bugcrowd/test-summary-buildkite-plugin/pull/3) 70 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:2.5.1-alpine 2 | 3 | # Fetch/install gems 4 | RUN mkdir -p /opt/gems 5 | COPY Gemfile Gemfile.lock /opt/gems/ 6 | WORKDIR /opt/gems 7 | RUN bundle install --deployment --without development 8 | 9 | ENV APP_DIR=/usr/src/app 10 | 11 | COPY . $APP_DIR 12 | RUN mkdir -p $APP_DIR/vendor && ln -s /opt/gems/vendor/bundle $APP_DIR/vendor/bundle 13 | 14 | WORKDIR $APP_DIR 15 | CMD ["./bin/run"] 16 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } 6 | 7 | gem 'bundler', '~> 1.16' 8 | gem 'haml' 9 | 10 | group :development, :test do 11 | gem 'rake', '~> 12.3' 12 | gem 'rspec', '~> 3.0' 13 | gem 'rspec_junit_formatter' 14 | gem 'rubocop' 15 | end 16 | 17 | group :development do 18 | gem 'commonmarker' 19 | end 20 | 21 | group :test do 22 | gem 'simplecov', require: false 23 | end 24 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | ast (2.4.0) 5 | commonmarker (0.17.9) 6 | ruby-enum (~> 0.5) 7 | concurrent-ruby (1.0.5) 8 | diff-lcs (1.3) 9 | docile (1.3.0) 10 | haml (5.0.4) 11 | temple (>= 0.8.0) 12 | tilt 13 | i18n (1.0.1) 14 | concurrent-ruby (~> 1.0) 15 | json (2.1.0) 16 | parallel (1.12.1) 17 | parser (2.5.1.0) 18 | ast (~> 2.4.0) 19 | powerpack (0.1.1) 20 | rainbow (3.0.0) 21 | rake (12.3.3) 22 | rspec (3.7.0) 23 | rspec-core (~> 3.7.0) 24 | rspec-expectations (~> 3.7.0) 25 | rspec-mocks (~> 3.7.0) 26 | rspec-core (3.7.1) 27 | rspec-support (~> 3.7.0) 28 | rspec-expectations (3.7.0) 29 | diff-lcs (>= 1.2.0, < 2.0) 30 | rspec-support (~> 3.7.0) 31 | rspec-mocks (3.7.0) 32 | diff-lcs (>= 1.2.0, < 2.0) 33 | rspec-support (~> 3.7.0) 34 | rspec-support (3.7.1) 35 | rspec_junit_formatter (0.4.1) 36 | rspec-core (>= 2, < 4, != 2.12.0) 37 | rubocop (0.55.0) 38 | parallel (~> 1.10) 39 | parser (>= 2.5) 40 | powerpack (~> 0.1) 41 | rainbow (>= 2.2.2, < 4.0) 42 | ruby-progressbar (~> 1.7) 43 | unicode-display_width (~> 1.0, >= 1.0.1) 44 | ruby-enum (0.7.2) 45 | i18n 46 | ruby-progressbar (1.9.0) 47 | simplecov (0.16.1) 48 | docile (~> 1.1) 49 | json (>= 1.8, < 3) 50 | simplecov-html (~> 0.10.0) 51 | simplecov-html (0.10.2) 52 | temple (0.8.0) 53 | tilt (2.0.8) 54 | unicode-display_width (1.3.2) 55 | 56 | PLATFORMS 57 | ruby 58 | 59 | DEPENDENCIES 60 | bundler (~> 1.16) 61 | commonmarker 62 | haml 63 | rake (~> 12.3) 64 | rspec (~> 3.0) 65 | rspec_junit_formatter 66 | rubocop 67 | simplecov 68 | 69 | BUNDLED WITH 70 | 1.16.2 71 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 tessereth 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Test Summary Buildkite Plugin 2 | 3 | A [Buildkite plugin](https://buildkite.com/docs/agent/v3/plugins) that adds a single annotation 4 | for all your test failures using 5 | [buildkite-agent annotate](https://buildkite.com/docs/agent/v3/cli-annotate). 6 | 7 | Supported formats: 8 | 9 | * JUnit 10 | * Checkstyle 11 | * [TAP](https://testanything.org)^ 12 | * Plain text files with one failure per line 13 | 14 | \^ Current TAP support is fairly limited. If you have an example TAP file that is not being interpreted correctly, 15 | feel free to open an issue or pull request. 16 | 17 | ## Example 18 | 19 | Upload test results as artifacts using any supported format. If you include the `$BUILDKITE_JOB_ID` in the path, 20 | a link to the build will be included in the annotation. 21 | Some examples: 22 | 23 | ```yaml 24 | steps: 25 | - label: rspec 26 | command: rspec 27 | parallelism: 10 28 | # With spec_helper.rb: 29 | # RSpec.configure do |config| 30 | # config.add_formatter('RspecJunitFormatter', "artifacts/rspec-#{ENV['BUILDKITE_JOB_ID']}.xml") 31 | # end 32 | artifact_paths: "artifacts/*" 33 | 34 | - label: eslint 35 | command: yarn run eslint -f checkstyle -o artifacts/eslint.xml 36 | artifact_paths: "artifacts/*" 37 | 38 | - label: ava 39 | command: bash -c "yarn --silent test --tap > artifacts/ava.tap" 40 | artifact_paths: "artifacts/*" 41 | 42 | - label: rubocop 43 | # The emacs format is plain text with one failure per line 44 | command: rubocop -f emacs -o artifacts/rubocop.txt 45 | artifact_paths: "artifacts/*" 46 | ``` 47 | 48 | Wait for all the tests to finish: 49 | 50 | ```yaml 51 | - wait: ~ 52 | continue_on_failure: true 53 | ``` 54 | 55 | Add a build step using the test-summary plugin: 56 | 57 | ```yaml 58 | - label: annotate 59 | plugins: 60 | - bugcrowd/test-summary#v1.11.0: 61 | inputs: 62 | - label: rspec 63 | artifact_path: artifacts/rspec* 64 | type: junit 65 | - label: eslint 66 | artifact_path: artifacts/eslint.xml 67 | type: checkstyle 68 | - label: ava 69 | artifact_path: artifacts/ava.tap 70 | type: tap 71 | - label: rubocop 72 | artifact_path: artifacts/rubocop.txt 73 | type: oneline 74 | formatter: 75 | type: details 76 | context: test-summary 77 | ``` 78 | 79 | See buildkite annotation of all the failures. There are some example annotations included below. 80 | 81 | ## Configuration 82 | 83 | ### Inputs 84 | 85 | The plugin takes a list of input sources. Each input source has: 86 | 87 | * `label:` the name used in the heading to identify the test group. 88 | * `artifact_path:` a glob used to download one or more artifacts. 89 | * `type:` one of `junit`, `checkstyle`, `tap` or `oneline`. 90 | * `encoding:` The file encoding to use. Defaults to `UTF-8`. 91 | * `strip_colors:` Remove ANSI color escape sequences. Defaults to `false`. 92 | * `crop:` (`oneline` type only) Number of lines to crop from the start and end of the file, 93 | to get around default headers and footers. Eg: 94 | 95 | ```yaml 96 | crop: 97 | start: 3 98 | end: 2 99 | ``` 100 | 101 | * `job_id_regex:` Ruby regular expression to extract the `job_id` from the artifact path. It must contain 102 | a named capture with the name `job_id`. Defaults to 103 | `(?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})`. 104 | 105 | #### Junit specific options: 106 | 107 | * `summary:` Customise how the summary is generated. Includes: 108 | * `format:` A ruby format string for converting the junit xml attributes 109 | into a summary. All attributes are available in `.` format. 110 | * `details_regex:` A ruby regular expression, run over the body text of each failure. Any named captures 111 | generated by this regex will be available to the format string. This is useful if some information is only 112 | available in the contents of the failure. Eg: 113 | 114 | ```yaml 115 | summary: 116 | format: '%{testsuites.name}: %{testsuite.name}: %{testcase.classname}: %{failure.message}%{error.message}' 117 | ``` 118 | 119 | ```yaml 120 | summary: 121 | format: '%{location}: %{testcase.name}' 122 | details_regex: '(?\S+:\d+)' 123 | ``` 124 | 125 | * `message:` Set this to false if the failure `message` attribute is not worth showing in the annotation. Defaults to `true`. 126 | * `details:` Set this to false if the body of the failure is not worth showing in the annotation. Defaults to `true`. 127 | 128 | ### Formatter 129 | 130 | There are two formatter types, `summary` and `details`. 131 | 132 | The `summary` formatter includes a single line for each failure. 133 | 134 | ![example summary annotation](doc/summary.png) 135 | 136 | The `details` formatter 137 | includes extra information about the failure in an accordion (if available). 138 | This is the default option. 139 | 140 | ![example details annotation](doc/details.png) 141 | 142 | Other formatter options are: 143 | 144 | * `show_first:` The number of failures to show before hiding the rest inside an accordion. 145 | If set to zero, all failures will be hidden by default. If set to a negative number, all failures 146 | will be shown. Defaults to 20. 147 | 148 | ### Other options 149 | 150 | * `context:` The Buildkite annotation context. Defaults to `test-summary`. 151 | * `style:` Set the annotation style. Defaults to `error`. 152 | * `fail_on_error:` Whether the command should return non-zero exit status on failure. Defaults to `false` so failing 153 | to annotate a build does not cause the entire pipeline to fail. 154 | * `run_without_docker:` Set the enviroment to run without docker. Defaults to `false`. 155 | 156 | ## Truncation 157 | 158 | Buildkite has a maximum annotation size of 100 kilobytes. If there are too many failures to fit within this limit, the 159 | plugin will truncate the failures for each input. 160 | 161 | ## Developing 162 | 163 | To run the tests: 164 | 165 | docker-compose run --rm test rspec 166 | 167 | If you have ruby set up, you can just run: 168 | 169 | bundle install 170 | rspec 171 | 172 | To generate sample markdown and HTML based on the files in `spec/sample_artifacts`: 173 | 174 | bin/run-dev 175 | 176 | Note: The sample HTML files use hardcoded references to buildkite assets. If the page is not displaying correctly, 177 | try updating the css files in [templates/test_layout.html.haml](templates/test_layout.html.haml) based on what 178 | buildkite is currently serving. 179 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | ### Release process 2 | 3 | 1. Update [version.rb](lib/test_summary_buildkite_plugin/version.rb) 4 | 2. Update version in README example 5 | 3. Update [CHANGELOG.md](./CHANGELOG.md) 6 | 4. Ensure screenshots are up to date 7 | 5. Push to github and ensure tests pass 8 | 7. `export NEXT_VERSION=vx.x.x` 9 | 6. `docker build -t bugcrowd/test-summary-buildkite-plugin:$NEXT_VERSION .` 10 | 7. `git tag --sign $NEXT_VERSION -m "Release $NEXT_VERSION"` 11 | 8. `docker push bugcrowd/test-summary-buildkite-plugin:$NEXT_VERSION` 12 | 9. `git push origin $NEXT_VERSION` 13 | 10. Copy changelog entry to github release notes 14 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'bundler/gem_tasks' 4 | require 'rspec/core/rake_task' 5 | 6 | RSpec::Core::RakeTask.new(:spec) 7 | 8 | task default: :spec 9 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require 'bundler/setup' 5 | require 'pathname' 6 | $LOAD_PATH.unshift(Pathname.new(__FILE__).dirname.dirname.join('lib').to_s) 7 | require 'test_summary_buildkite_plugin' 8 | 9 | # You can add fixtures and/or initialization code here to make experimenting 10 | # with your gem easier. You can also use a different console, if you like. 11 | 12 | # (If you use this, don't forget to add pry to your Gemfile!) 13 | # require "pry" 14 | # Pry.start 15 | 16 | require 'irb' 17 | IRB.start(__FILE__) 18 | -------------------------------------------------------------------------------- /bin/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require 'bundler/setup' 5 | require 'pathname' 6 | $LOAD_PATH.unshift(Pathname.new(__FILE__).dirname.dirname.join('lib').to_s) 7 | 8 | require 'test_summary_buildkite_plugin' 9 | 10 | TestSummaryBuildkitePlugin.run 11 | -------------------------------------------------------------------------------- /bin/run-dev: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require 'bundler/setup' 5 | require 'pathname' 6 | $LOAD_PATH.unshift(Pathname.new(__FILE__).dirname.dirname.join('lib').to_s) 7 | 8 | require 'commonmarker' 9 | require 'yaml' 10 | require 'test_summary_buildkite_plugin' 11 | 12 | # Stub out things that don't work in a development environment 13 | 14 | TestSummaryBuildkitePlugin::Input.send(:remove_const, 'WORKDIR') 15 | TestSummaryBuildkitePlugin::Input::WORKDIR = '.' 16 | 17 | module TestSummaryBuildkitePlugin 18 | class Agent 19 | def run(*args, stdin: nil) 20 | log(args, stdin: stdin) 21 | if args.first == 'annotate' 22 | context = args[2] 23 | content = CommonMarker.render_html(stdin) 24 | html = HamlRender.render('test_layout', content: content) 25 | FileUtils.mkdir_p('tmp') 26 | out = Pathname.new('tmp').join(context + '.html') 27 | out.open('w') { |file| file.write(html) } 28 | puts("Sample HTML written to file://#{out.realdirpath}") 29 | end 30 | end 31 | end 32 | end 33 | 34 | # Load test config from the pipeline 35 | pipeline_sample = YAML.load_file('.buildkite/pipeline.yml') 36 | 37 | pipeline_sample['steps'].each do |step| 38 | next unless step['plugins']&.any? { |plugin| plugin.find { |k, _| k.include?('test-summary') } } 39 | 40 | # Roughly match the formatting that buildkite gives 41 | ENV['BUILDKITE_PLUGINS'] = step['plugins'].to_json 42 | TestSummaryBuildkitePlugin.run 43 | end 44 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /doc/details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bugcrowd/test-summary-buildkite-plugin/17ca15118a1959e0ba636fc1a5f036607e90f837/doc/details.png -------------------------------------------------------------------------------- /doc/summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bugcrowd/test-summary-buildkite-plugin/17ca15118a1959e0ba636fc1a5f036607e90f837/doc/summary.png -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | test: 4 | build: . 5 | volumes: 6 | - ./artifacts:/usr/src/app/artifacts 7 | -------------------------------------------------------------------------------- /hooks/command: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eEuo pipefail 4 | 5 | DOCKER_REPO=bugcrowd/test-summary-buildkite-plugin 6 | 7 | on_failure() { 8 | echo "Command failed with exit status: $?" 9 | if [[ "${BUILDKITE_PLUGIN_TEST_SUMMARY_FAIL_ON_ERROR:-false}" != "true" ]]; then 10 | echo "Suppressing failure so pipeline can continue (if you do not want this behaviour, set fail_on_error to true)" 11 | exit 0 12 | fi 13 | } 14 | 15 | trap on_failure ERR 16 | 17 | # cd to plugin directory 18 | cd "$( dirname "${BASH_SOURCE[0]}" )/.." 19 | 20 | if [[ ${BUILDKITE_PLUGIN_TEST_SUMMARY_RUN_WITHOUT_DOCKER:-false} = true ]]; then 21 | ./bin/setup 22 | ./bin/run 23 | else 24 | TAG=$(git describe --tags --exact-match 2> /dev/null || true) 25 | 26 | if [[ -n "$TAG" ]]; then 27 | echo "Found tag $TAG, pulling from docker hub" 28 | IMAGE="$DOCKER_REPO:$TAG" 29 | docker pull "$IMAGE" 30 | else 31 | echo "No tag found, building image locally" 32 | IMAGE=test-summary:$BUILDKITE_JOB_ID 33 | docker build -t "$IMAGE" . 34 | fi 35 | 36 | docker run --rm \ 37 | --mount type=bind,src=$(which buildkite-agent),dst=/usr/bin/buildkite-agent \ 38 | --mount type=bind,src=/tmp,dst=/tmp \ 39 | -e BUILDKITE_BUILD_ID -e BUILDKITE_JOB_ID -e BUILDKITE_PLUGINS \ 40 | -e BUILDKITE_AGENT_ID -e BUILDKITE_AGENT_ACCESS_TOKEN -e BUILDKITE_AGENT_ENDPOINT \ 41 | -e HTTP_PROXY -e HTTPS_PROXY \ 42 | $IMAGE 43 | fi 44 | -------------------------------------------------------------------------------- /lib/test_summary_buildkite_plugin.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'json' 4 | 5 | require 'test_summary_buildkite_plugin/agent' 6 | require 'test_summary_buildkite_plugin/failure' 7 | require 'test_summary_buildkite_plugin/formatter' 8 | require 'test_summary_buildkite_plugin/haml_render' 9 | require 'test_summary_buildkite_plugin/input' 10 | require 'test_summary_buildkite_plugin/runner' 11 | require 'test_summary_buildkite_plugin/tap' 12 | require 'test_summary_buildkite_plugin/truncater' 13 | require 'test_summary_buildkite_plugin/utils' 14 | require 'test_summary_buildkite_plugin/version' 15 | 16 | module TestSummaryBuildkitePlugin 17 | def self.run 18 | plugins = JSON.parse(ENV.fetch('BUILDKITE_PLUGINS'), symbolize_names: true) 19 | # plugins is an array of hashes, keyed by # 20 | options = plugins.find { |k, _| k.to_s.include?('test-summary') }.values.first 21 | Runner.new(options).run 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/test_summary_buildkite_plugin/agent.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'English' 4 | require 'forwardable' 5 | require 'singleton' 6 | 7 | module TestSummaryBuildkitePlugin 8 | class Agent 9 | include Singleton 10 | 11 | class << self 12 | extend Forwardable 13 | def_delegators :instance, :run 14 | end 15 | 16 | attr_accessor :stub 17 | 18 | def run(*args, stdin: nil) 19 | log(args, stdin: stdin) 20 | cmd = %w[buildkite-agent] + args 21 | IO.popen(cmd, 'w+') do |io| 22 | io.write(stdin) if stdin 23 | io.close_write 24 | puts io.read 25 | end 26 | if $CHILD_STATUS.exitstatus != 0 27 | raise CommandFailed, "Command '#{cmd.join(' ')}' failed (exit status: #{$CHILD_STATUS.exitstatus})" 28 | end 29 | end 30 | 31 | def log(args, stdin: nil) 32 | puts('$ buildkite-agent ' + args.join(' ')) 33 | if stdin 34 | puts('# with stdin:') 35 | puts(stdin) 36 | end 37 | end 38 | 39 | class CommandFailed < StandardError; end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/test_summary_buildkite_plugin/failure.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module TestSummaryBuildkitePlugin 4 | module Failure 5 | # All failure classes should have #summary and #details methods 6 | class Base 7 | attr_accessor :job_id 8 | 9 | def strip_colors 10 | instance_variables.each do |var| 11 | value = instance_variable_get(var) 12 | instance_variable_set(var, value.gsub(/\\e\[[\d;]+m/, '')) if value.is_a?(String) 13 | end 14 | end 15 | end 16 | 17 | class Unstructured < Base 18 | attr_reader :summary 19 | 20 | def initialize(summary) 21 | @summary = summary 22 | end 23 | 24 | def details; end 25 | 26 | def message; end 27 | end 28 | 29 | class Structured < Base 30 | attr_accessor :summary, :message, :details 31 | 32 | def initialize(summary:, message: nil, details: nil) 33 | @summary = summary 34 | @message = message 35 | @details = details 36 | end 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/test_summary_buildkite_plugin/formatter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'haml' 4 | 5 | module TestSummaryBuildkitePlugin 6 | class Formatter 7 | def self.create(**options) 8 | options[:type] ||= 'details' 9 | type = options[:type].to_sym 10 | raise "Unknown type: #{type}" unless TYPES.key?(type) 11 | TYPES[type].new(options) 12 | end 13 | 14 | class Base 15 | attr_reader :options 16 | 17 | def initialize(options = {}) 18 | @options = options || {} 19 | end 20 | 21 | def markdown(input) 22 | return nil if input.failures.count.zero? 23 | [heading(input), input_markdown(input), footer(input)].compact.join("\n\n") 24 | end 25 | 26 | protected 27 | 28 | def input_markdown(input) 29 | if show_first.negative? || show_first >= include_failures(input).count 30 | failures_markdown(include_failures(input)) 31 | elsif show_first.zero? 32 | details('Show failures', failures_markdown(include_failures(input))) 33 | else 34 | failures_markdown(include_failures(input)[0...show_first]) + 35 | details('Show additional failures', failures_markdown(include_failures(input)[show_first..-1])) 36 | end 37 | end 38 | 39 | def failures_markdown(failures) 40 | render_template('failures', failures: failures) 41 | end 42 | 43 | def heading(input) 44 | count = input.failures.count 45 | show_count = include_failures(input).count 46 | s = "##### #{input.label}: #{count} failure#{'s' unless count == 1}" 47 | s += "\n\n_Including first #{show_count} failures_" if show_count < count 48 | s 49 | end 50 | 51 | def footer(input) 52 | job_ids = input.failures.map(&:job_id).uniq.reject(&:nil?) 53 | render_template('footer', job_ids: job_ids) 54 | end 55 | 56 | def show_first 57 | options[:show_first] || 20 58 | end 59 | 60 | def details(summary, contents) 61 | "
\n#{summary}\n#{contents}\n
" 62 | end 63 | 64 | def type 65 | options[:type] || 'details' 66 | end 67 | 68 | def truncate 69 | options[:truncate] 70 | end 71 | 72 | def include_failures(input) 73 | if truncate 74 | input.failures[0...truncate] 75 | else 76 | input.failures 77 | end 78 | end 79 | 80 | def render_template(name, params) 81 | # In CommonMark (used by buildkite), most html block elements are terminated by a blank line 82 | # So we need to ensure we don't have any of those in the middle of our html 83 | # 84 | # See https://spec.commonmark.org/0.28/#html-blocks 85 | HamlRender.render(name, params, folder: type)&.gsub(/\n\n+/, "\n \n") 86 | end 87 | end 88 | 89 | class Summary < Base; end 90 | 91 | class Details < Base; end 92 | 93 | TYPES = { 94 | summary: Formatter::Summary, 95 | details: Formatter::Details 96 | }.freeze 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /lib/test_summary_buildkite_plugin/haml_render.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module TestSummaryBuildkitePlugin 4 | class HamlRender 5 | def self.render(name, params, folder: nil) 6 | filename = %W[templates/#{folder}/#{name}.html.haml templates/#{name}.html.haml].find { |f| File.exist?(f) } 7 | if filename 8 | engine = Haml::Engine.new(File.read(filename), escape_html: true) 9 | engine.render(Object.new, params) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/test_summary_buildkite_plugin/input.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # We don't use nokogiri because we use an alpine-based docker image 4 | # And adding the required dependencies triples the size of the image 5 | require 'rexml/document' 6 | 7 | module TestSummaryBuildkitePlugin 8 | module Input 9 | WORKDIR = 'tmp/test-summary' 10 | DEFAULT_JOB_ID_REGEX = /(?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/ 11 | 12 | def self.create(type:, **options) 13 | type = type.to_sym 14 | raise StandardError, "Unknown file type: #{type}" unless TYPES.key?(type) 15 | TYPES[type].new(options) 16 | end 17 | 18 | class Base 19 | attr_reader :label, :artifact_path, :options 20 | 21 | def initialize(label:, artifact_path:, **options) 22 | @label = label 23 | @artifact_path = artifact_path 24 | @options = options 25 | end 26 | 27 | def failures 28 | @failures ||= begin 29 | f = files.map { |filename| filename_to_failures(filename) }.flatten 30 | f.each(&:strip_colors) if options[:strip_colors] 31 | f.sort_by(&:summary) 32 | end 33 | end 34 | 35 | protected 36 | 37 | def files 38 | @files ||= begin 39 | FileUtils.mkpath(WORKDIR) 40 | Agent.run('artifact', 'download', artifact_path, WORKDIR) 41 | Dir.glob("#{WORKDIR}/#{artifact_path}") 42 | rescue Agent::CommandFailed => err 43 | handle_error(err) 44 | [] 45 | end 46 | end 47 | 48 | def read(filename) 49 | File.read(filename).force_encoding(encoding) 50 | end 51 | 52 | def encoding 53 | @options[:encoding] || 'UTF-8' 54 | end 55 | 56 | def fail_on_error 57 | @options[:fail_on_error] || false 58 | end 59 | 60 | def filename_to_failures(filename) 61 | file_contents_to_failures(read(filename)).each { |failure| failure.job_id = job_id(filename) } 62 | rescue StandardError => err 63 | handle_error(err) 64 | [] 65 | end 66 | 67 | def job_id(filename) 68 | filename.match(job_id_regex)&.named_captures&.fetch('job_id', nil) 69 | end 70 | 71 | def job_id_regex 72 | if @options[:job_id_regex] 73 | r = Regexp.new(@options[:job_id_regex]) 74 | raise 'Job id regex must have a job_id named capture' unless r.names.include?('job_id') 75 | r 76 | else 77 | DEFAULT_JOB_ID_REGEX 78 | end 79 | end 80 | 81 | def handle_error(err) 82 | if fail_on_error 83 | raise err 84 | else 85 | Utils.log_error(err) 86 | end 87 | end 88 | end 89 | 90 | class OneLine < Base 91 | def file_contents_to_failures(str) 92 | str.split("\n")[crop.start..crop.end] 93 | .reject(&:empty?) 94 | .map { |line| Failure::Unstructured.new(line) } 95 | end 96 | 97 | private 98 | 99 | def crop 100 | @crop ||= OpenStruct.new( 101 | start: options.dig(:crop, :start) || 0, 102 | end: -1 - (options.dig(:crop, :end) || 0) 103 | ) 104 | end 105 | end 106 | 107 | class JUnit < Base 108 | def file_contents_to_failures(str) 109 | xml = REXML::Document.new(str) 110 | xml.elements.enum_for(:each, '//testcase').each_with_object([]) do |testcase, failures| 111 | testcase.elements.each('failure | error') do |failure| 112 | failures << Failure::Structured.new( 113 | summary: summary(failure), 114 | message: message(failure), 115 | details: details(failure) 116 | ) 117 | end 118 | end 119 | end 120 | 121 | def summary(failure) 122 | data = attributes(failure) 123 | if summary_format 124 | summary_format % data 125 | else 126 | name = data[:'testcase.name'] 127 | file = data[:'testcase.file'] 128 | class_name = data[:'testcase.classname'] 129 | location = if !file.nil? && !file.empty? 130 | "#{file}: " 131 | elsif !class_name.nil? && !class_name.empty? && class_name != name 132 | "#{class_name}: " 133 | end 134 | "#{location}#{name}" 135 | end 136 | end 137 | 138 | def attributes(failure) 139 | # If elements are used in the format string but don't exist in the map, pretend they're blank 140 | acc = Hash.new('') 141 | elem = failure 142 | until elem.parent.nil? 143 | elem.attributes.each do |attr_name, attr_value| 144 | acc["#{elem.name}.#{attr_name}".to_sym] = attr_value 145 | end 146 | elem = elem.parent 147 | end 148 | acc.merge(detail_attributes(failure)) 149 | end 150 | 151 | def detail_attributes(failure) 152 | matches = details_regex&.match(details(failure))&.named_captures || {} 153 | # need to symbolize keys 154 | matches.each_with_object({}) do |(key, value), acc| 155 | acc[key.to_sym] = value 156 | end 157 | end 158 | 159 | def details(failure) 160 | if options.fetch(:details, true) 161 | # gets all text elements that are direct children (includes CDATA), and use the unescaped values 162 | failure.texts.map(&:value).join('').strip 163 | end 164 | end 165 | 166 | def message(failure) 167 | failure.attributes['message']&.to_s if options.fetch(:message, true) 168 | end 169 | 170 | def summary_format 171 | @summary_format ||= options.dig(:summary, :format) 172 | end 173 | 174 | def details_regex 175 | @details_regex ||= begin 176 | regex_str = options.dig(:summary, :details_regex) 177 | Regexp.new(regex_str) if regex_str 178 | end 179 | end 180 | end 181 | 182 | class Tap < Base 183 | def file_contents_to_failures(tap) 184 | suite = ::TestSummaryBuildkitePlugin::Tap::Parser.new(tap).parse 185 | suite.tests.select { |x| !x.passed && !x.todo && !x.skipped }.map do |x| 186 | Failure::Structured.new( 187 | summary: x.description, 188 | details: x.yaml || x.diagnostic 189 | ) 190 | end 191 | end 192 | end 193 | 194 | class Checkstyle < Base 195 | def file_contents_to_failures(str) 196 | xml = REXML::Document.new(str) 197 | xml.elements.enum_for(:each, '//file').flat_map do |file| 198 | filename = file.attribute('name').value 199 | 200 | file.elements.map do |error| 201 | Failure::Structured.new( 202 | summary: summary(filename, error), 203 | details: error.attribute('source').value 204 | ) 205 | end 206 | end 207 | end 208 | 209 | def summary(filename, error) 210 | severity = error.attribute('severity')&.value 211 | line = error.attribute('line')&.value 212 | column = error.attribute('column')&.value 213 | location = [filename, line, column].compact.join(':') 214 | message = error.attribute('message')&.value 215 | 216 | "[#{severity}] #{location}: #{message}" 217 | end 218 | end 219 | 220 | TYPES = { 221 | oneline: Input::OneLine, 222 | junit: Input::JUnit, 223 | tap: Input::Tap, 224 | checkstyle: Input::Checkstyle 225 | }.freeze 226 | end 227 | end 228 | -------------------------------------------------------------------------------- /lib/test_summary_buildkite_plugin/runner.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module TestSummaryBuildkitePlugin 4 | class Runner 5 | MAX_MARKDOWN_SIZE = 100_000 6 | 7 | attr_reader :options 8 | 9 | def initialize(options) 10 | @options = options 11 | end 12 | 13 | def run 14 | markdown = Truncater.new( 15 | max_size: MAX_MARKDOWN_SIZE, 16 | inputs: inputs, 17 | formatter_opts: options[:formatter], 18 | fail_on_error: fail_on_error 19 | ).markdown 20 | if markdown.nil? || markdown.empty? 21 | puts('No errors found! 🎉') 22 | else 23 | annotate(markdown) 24 | end 25 | end 26 | 27 | def annotate(markdown) 28 | Agent.run('annotate', '--context', context, '--style', style, stdin: markdown) 29 | end 30 | 31 | def inputs 32 | @inputs ||= options[:inputs].map { |opts| Input.create(opts.merge(fail_on_error: fail_on_error)) } 33 | end 34 | 35 | def context 36 | options[:context] || 'test-summary' 37 | end 38 | 39 | def style 40 | options[:style] || 'error' 41 | end 42 | 43 | def fail_on_error 44 | options[:fail_on_error] || false 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/test_summary_buildkite_plugin/tap.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module TestSummaryBuildkitePlugin 4 | # Parses most of the TAP protocol, assuming the inputs are sane. 5 | # 6 | # The specification is at https://testanything.org. This parses both 7 | # version 12 and version 13. 8 | # 9 | # Notable omissions: 10 | # 11 | # * Test numbering and the planned number of tests are ignored. 12 | # * "Bail out!" is ignored. 13 | # 14 | # Disclaimer: 15 | # 16 | # This works about as well as you'd expect a hand-rolled parser made of 17 | # regular expressions to work. Use at your own risk, pull requests welcome. 18 | # 19 | # TODO: Use a proper grammar and parser rather than regexes. 20 | class Tap 21 | class Suite 22 | attr_accessor :tests, :version 23 | 24 | def initialize 25 | self.version = 12 26 | self.tests = [] 27 | end 28 | end 29 | 30 | Test = Struct.new(:passed, :description, :directive, :todo, :skipped, :diagnostic, :yaml, 31 | keyword_init: true) 32 | 33 | class Parser 34 | PATTERNS = { 35 | plan: /^(?\d+)\.\.(?\d+)/, 36 | test: 37 | /^(?not )?ok(? \d+)?(?[^#]*)(#\s*(?((?TODO)|(?SKIP))?.*))?/i, 38 | comment: /^#(?.*)$/, 39 | yaml_start: /^\s+---/, 40 | yaml_end: /^\s+\.\.\./, 41 | version: /^TAP version (?\d+)/i 42 | }.freeze 43 | 44 | attr_reader :text 45 | attr_reader :suite 46 | 47 | def initialize(text) 48 | @text = text 49 | @suite = Suite.new 50 | @current_diagnostic = [] 51 | @current_yaml = [] 52 | @in_yaml = [] 53 | end 54 | 55 | def parse # rubocop:disable Metrics/MethodLength 56 | text.split("\n").each do |line| 57 | type, match = type(line) 58 | case type 59 | when :test 60 | save_previous_blocks 61 | suite.tests.push(to_test(match)) 62 | when :version 63 | suite.version = match['version'].to_i 64 | when :plan 65 | # we currently have no use for the 1..x info 66 | nil 67 | when :comment 68 | @current_diagnostic.push(match['comment']) 69 | when :yaml_start 70 | @in_yaml = true 71 | when :yaml_end 72 | @in_yaml = false 73 | else 74 | @current_yaml.push(line) if @in_yaml 75 | # as per the spec, we just ignore anything else we don't recognise 76 | end 77 | end 78 | save_previous_blocks 79 | suite 80 | end 81 | 82 | private 83 | 84 | def type(line) 85 | PATTERNS.each do |name, regex| 86 | match = regex.match(line) 87 | return name, match if match 88 | end 89 | [:unknown, nil] 90 | end 91 | 92 | def to_test(match) 93 | Test.new( 94 | passed: !match['not'], 95 | description: description(match), 96 | directive: match['directive'], 97 | todo: match['todo'], 98 | skipped: match['skip'] 99 | ) 100 | end 101 | 102 | def description(match) 103 | # There's a convention to put a ' - ' between the test number and the description 104 | # We strip that for better readability 105 | match['description'].strip.gsub(/^- /, '') 106 | end 107 | 108 | def save_previous_blocks 109 | last_test = suite.tests.last 110 | if last_test 111 | last_test.diagnostic = normalize_multiline(@current_diagnostic) 112 | last_test.yaml = normalize_multiline(@current_yaml) 113 | end 114 | @current_diagnostic = [] 115 | @current_yaml = [] 116 | @in_yaml = false 117 | end 118 | 119 | def normalize_multiline(lines) 120 | if lines.empty? 121 | nil 122 | else 123 | indent = lines.first.match(/(\s*)/)[1].length 124 | lines.map { |line| line[indent..-1] }.join("\n") 125 | end 126 | end 127 | end 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /lib/test_summary_buildkite_plugin/truncater.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module TestSummaryBuildkitePlugin 4 | class Truncater 5 | attr_reader :max_size, :inputs, :formatter_opts, :fail_on_error 6 | 7 | def initialize(max_size:, inputs:, formatter_opts: {}, fail_on_error: false) 8 | @max_size = max_size 9 | @inputs = inputs 10 | @formatter_opts = formatter_opts || {} 11 | @fail_on_error = fail_on_error 12 | @_input_markdown = {} 13 | @_formatter = {} 14 | end 15 | 16 | def markdown 17 | requested = markdown_with_truncation(nil) 18 | if requested.empty? || requested.bytesize < max_size 19 | # we can use it as-is, no need to truncate 20 | return requested 21 | end 22 | puts "Markdown is too large (#{requested.bytesize} B > #{max_size} B), truncating" 23 | 24 | # See http://ruby-doc.org/core/Array.html#method-i-bsearch 25 | # 26 | # The block must return false for every value before the result 27 | # and true for the result and every value after 28 | best_truncate = (0..max_truncate).to_a.reverse.bsearch do |truncate| 29 | puts "Test truncating to #{truncate}: bytesize=#{markdown_with_truncation(truncate).bytesize}" 30 | markdown_with_truncation(truncate).bytesize <= max_size 31 | end 32 | if best_truncate.nil? 33 | # If we end up here, we failed to find a valid truncation value 34 | # ASAICT this should never happen but if it does, something is very wrong 35 | # so ask the user to let us know 36 | return bug_report_message 37 | end 38 | puts "Optimal truncation: #{best_truncate}" 39 | markdown_with_truncation(best_truncate) 40 | end 41 | 42 | private 43 | 44 | def formatter(truncate) 45 | @_formatter[truncate] ||= Formatter.create(formatter_opts.merge(truncate: truncate)) 46 | end 47 | 48 | def input_markdown(input, truncate = nil) 49 | @_input_markdown[[input, truncate]] ||= formatter(truncate).markdown(input) 50 | rescue StandardError => e 51 | if fail_on_error 52 | raise 53 | else 54 | Utils.log_error(e) 55 | nil 56 | end 57 | end 58 | 59 | def markdown_with_truncation(truncate) 60 | inputs.map { |input| input_markdown(input, truncate) }.compact.join("\n\n") 61 | end 62 | 63 | def max_truncate 64 | @max_truncate ||= inputs.map(&:failures).map(&:count).max 65 | end 66 | 67 | def bug_report_message 68 | puts 69 | puts 'Optimization failed 😱' 70 | puts 'Please report this to https://github.com/bugcrowd/test-summary-buildkite-plugin/issues' 71 | puts 'with the test log above and the details below.' 72 | puts JSON.pretty_generate(diagnostics) 73 | HamlRender.render('truncater_exception', {}) 74 | end 75 | 76 | def diagnostics 77 | { 78 | max_size: max_size, 79 | formatter: formatter_opts, 80 | inputs: inputs.map do |input| 81 | { 82 | type: input.class, 83 | failure_count: input.failures.count, 84 | markdown_bytesize: input_markdown(input, nil)&.bytesize 85 | } 86 | end 87 | } 88 | end 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /lib/test_summary_buildkite_plugin/utils.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module TestSummaryBuildkitePlugin 4 | module Utils 5 | def log_error(err) 6 | puts "#{err.class}: #{err.message}\n\n#{err.backtrace.join("\n")}" 7 | end 8 | 9 | module_function :log_error 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/test_summary_buildkite_plugin/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module TestSummaryBuildkitePlugin 4 | VERSION = '1.11.0' 5 | end 6 | -------------------------------------------------------------------------------- /plugin.yml: -------------------------------------------------------------------------------- 1 | name: Test Summary 2 | description: Collates test results as a buildkite annotation 3 | author: "@tessereth" 4 | requirements: 5 | - docker 6 | configuration: 7 | properties: 8 | inputs: 9 | type: array 10 | items: 11 | type: object 12 | properties: 13 | label: 14 | type: string 15 | artifact_path: 16 | type: string 17 | type: 18 | type: string 19 | enum: 20 | - junit 21 | - checkstyle 22 | - tap 23 | - oneline 24 | encoding: 25 | type: string 26 | strip_colors: 27 | type: boolean 28 | message: 29 | type: boolean 30 | details: 31 | type: boolean 32 | crop: 33 | type: object 34 | properties: 35 | start: 36 | type: number 37 | end: 38 | type: number 39 | summary: 40 | type: object 41 | properties: 42 | format: 43 | type: string 44 | details_regex: 45 | type: string 46 | additionalProperties: false 47 | required: 48 | - label 49 | - artifact_path 50 | - type 51 | additionalProperties: false 52 | formatter: 53 | type: object 54 | properties: 55 | type: 56 | type: string 57 | enum: 58 | - summary 59 | - details 60 | show_first: 61 | type: number 62 | additionalProperties: false 63 | context: 64 | type: string 65 | style: 66 | type: string 67 | fail_on_error: 68 | type: boolean 69 | run_without_docker: 70 | type: boolean 71 | required: 72 | - inputs 73 | additionalProperties: false 74 | -------------------------------------------------------------------------------- /spec/sample_artifacts/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /spec/sample_artifacts/eslint-00112233-0011-0011-0011-001122334455.txt: -------------------------------------------------------------------------------- 1 | /usr/src/app/javascript/src/foo.js:8:25: Delete `⏎···` [Error/prettier/prettier] 2 | /usr/src/app/javascript/src/bar.js::4:10: 'List' is defined but never used. [Error/no-unused-vars] 3 | 4 | 1 problem 5 | -------------------------------------------------------------------------------- /spec/sample_artifacts/example.tap: -------------------------------------------------------------------------------- 1 | TAP version 13 2 | ok 1 - retrieving servers from the database 3 | # need to ping 6 servers 4 | ok 2 - pinged diamond 5 | ok 3 - pinged ruby 6 | not ok 4 - pinged sapphire 7 | --- 8 | message: 'hostname "sapphire" unknown' 9 | severity: fail 10 | ... 11 | ok 5 - pinged onyx 12 | not ok 6 - pinged quartz 13 | --- 14 | message: 'timeout' 15 | severity: fail 16 | ... 17 | ok 7 - pinged gold 18 | 1..7 19 | -------------------------------------------------------------------------------- /spec/sample_artifacts/rspec-01234567-0011-0011-0011-001122334455.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Failure/Error: fail('awesomesauce') 9 | 10 | RuntimeError: 11 | awesomesauce 12 | ./spec/features/sign_in_out_spec.rb:27:in `block (3 levels) in <top (required)>' 13 | ./spec/features/sign_in_out_spec.rb:41:in `block (3 levels) in <top (required)>' 14 | 15 | 16 | -------------------------------------------------------------------------------- /spec/sample_artifacts/rspec-76543210-0011-0011-0011-001122334455.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Failure/Error: fail('whatever') 4 | 5 | RuntimeError: 6 | whatever 7 | ./spec/features/test_things_spec.rb:5:in `block (3 levels) in <top (required)>' 8 | 9 | -------------------------------------------------------------------------------- /spec/sample_artifacts/rspec-aabbccdd-0011-0011-0011-001122334455.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Failure/Error: \e[0m\e[32mit\e[0m { \e[32mexpect\e[0m(url).to be_allowed_url } 10 | expected http://foo.example.com to be allowed by ["*.foo.example.com"] with resource_scheme http 11 | ./spec/lib/url_whitelist_spec.rb:96:in `block (4 levels) in <top (required)>' 12 | Failure/Error: \e[0m\e[32mit\e[0m { \e[32mexpect\e[0m(url).not_to be_allowed_url } 13 | expected http://bar.foo.example.com not to be allowed by ["*.foo.example.com"] with resource_scheme http 14 | ./spec/lib/url_whitelist_spec.rb:92:in `block (4 levels) in <top (required)>' 15 | Failure/Error: \e[0m\e[32mit\e[0m { \e[32mexpect\e[0m(url).not_to be_allowed_url } 16 | expected http://baz.bar.foo.example.com not to be allowed by ["*.foo.example.com"] with resource_scheme http 17 | ./spec/lib/url_whitelist_spec.rb:92:in `block (4 levels) in <top (required)>' 18 | Failure/Error: \e[0m\e[32mit\e[0m { \e[32mexpect\e[0m(url).to be_allowed_url } 19 | expected http://example.com to be allowed by ["*.foo.example.com"] with resource_scheme http 20 | ./spec/lib/url_whitelist_spec.rb:96:in `block (4 levels) in <top (required)>' 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /spec/sample_artifacts/rubocop.txt: -------------------------------------------------------------------------------- 1 | /Users/foo/test-summary-buildkite-plugin/lib/test_summary_buildkite_plugin/agent.rb:22:7: C: Style/GuardClause: Use a guard clause instead of wrapping the code inside a conditional expression. 2 | /Users/foo/test-summary-buildkite-plugin/lib/test_summary_buildkite_plugin/formatter.rb:24:37: C: Style/Semicolon: Do not use semicolons to terminate expressions. 3 | /Users/foo/test-summary-buildkite-plugin/lib/test_summary_buildkite_plugin/formatter.rb:36:1: C: Layout/EmptyLinesAroundClassBody: Extra empty line detected at class body beginning. 4 | -------------------------------------------------------------------------------- /spec/sample_artifacts/version_12.tap: -------------------------------------------------------------------------------- 1 | # Started test HTTPS Server at localhost:36369 2 | # Starting Homeserver using SyTest::HomeserverFactory::Synapse=HASH(0x562459254088) 3 | # Starting server-0 4 | # Clearing Pg database pg1 on 'localhost' 5 | #** DBI connect('dbname=sytest_template;host=localhost','postgres',...) failed: FATAL: database "sytest_template" does not exist at lib/SyTest/Homeserver.pm line 371. 6 | # Generating config for port 8800 7 | # Creating config for server 0 with command /venv/bin/python -m coverage run --rcfile=/src/.coveragerc -m synapse.app.homeserver --config-path /work/server-0/config.yaml --server-name localhost:8800 --generate-config --report-stats=no 8 | Config file '/work/server-0/config.yaml' already exists. Generating any missing config files. 9 | Generating signing key file /work/server-0/localhost:8800.signing.key 10 | # Starting server 0 for port 8800 with command /venv/bin/python -m coverage run --rcfile=/src/.coveragerc -m synapse.app.homeserver --config-path /work/server-0/config.yaml --server-name localhost:8800 11 | # Connecting to server 8800 12 | # Connected to server 8800 13 | # Started server-0 14 | ok 1 GET /register yields a set of flows 15 | ok 2 POST /register can create a user 16 | ok 3 POST /register downcases capitals in usernames 17 | ok 4 POST /register returns the same device_id as that in the request 18 | ok 5 POST /register rejects registration of usernames with '!' 19 | ok 6 POST /register rejects registration of usernames with '"' 20 | ok 7 POST /register rejects registration of usernames with ':' 21 | ok 8 POST /register rejects registration of usernames with '?' 22 | ok 9 POST /register rejects registration of usernames with '\' 23 | ok 10 POST /register rejects registration of usernames with '@' 24 | ok 11 POST /register rejects registration of usernames with '[' 25 | ok 12 POST /register rejects registration of usernames with ']' 26 | ok 13 POST /register rejects registration of usernames with '{' 27 | ok 14 POST /register rejects registration of usernames with '|' 28 | ok 15 POST /register rejects registration of usernames with '}' 29 | ok 16 POST /register rejects registration of usernames with '£' 30 | ok 17 POST /register rejects registration of usernames with 'é' 31 | ok 18 POST /register rejects registration of usernames with '\n' 32 | ok 19 POST /register rejects registration of usernames with ''' 33 | ok 20 POST /r0/admin/register with shared secret 34 | ok 21 POST /r0/admin/register admin with shared secret 35 | ok 22 POST /r0/admin/register with shared secret downcases capitals 36 | ok 23 POST /r0/admin/register with shared secret disallows symbols 37 | ok 24 POST rejects invalid utf-8 in JSON 38 | ok 25 GET /login yields a set of flows 39 | ok 26 POST /login can log in as a user 40 | ok 27 POST /login returns the same device_id as that in the request 41 | ok 28 POST /login can log in as a user with just the local part of the id 42 | ok 29 POST /login as non-existing user is rejected 43 | ok 30 POST /login wrong password is rejected 44 | ok 31 GET /events initially 45 | ok 32 GET /initialSync initially 46 | ok 33 Version responds 200 OK with valid structure 47 | ok 34 PUT /profile/:user_id/displayname sets my name 48 | ok 35 GET /profile/:user_id/displayname publicly accessible 49 | ok 36 PUT /profile/:user_id/avatar_url sets my avatar 50 | ok 37 GET /profile/:user_id/avatar_url publicly accessible 51 | ok 38 GET /device/{deviceId} 52 | ok 39 GET /device/{deviceId} gives a 404 for unknown devices 53 | ok 40 GET /devices 54 | ok 41 PUT /device/{deviceId} updates device fields 55 | ok 42 PUT /device/{deviceId} gives a 404 for unknown devices 56 | not ok 43 DELETE /device/{deviceId} 57 | # Started: 2019-07-31 19:41:12.645 58 | # Ended: 2019-07-31 19:41:12.730 59 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/devices/login_device?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYWNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTExOmxvY2FsaG9zdDo4ODAwCjAwMTZjaWQgdHlwZSA9IGFjY2VzcwowMDIxY2lkIG5vbmNlID0gNzZ6Jk5EUSZFJmlidS5HUAowMDJmc2lnbmF0dXJlICuiqWxlyiv84gkjMdsHH8MTIBhIEXd42CUfRBw7BwQSCg ) from DELETE https://localhost:8800/_matrix/client/r0/devices/login_device?... 60 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 61 | # 0.021123: Registered new user @anon-20190731_194105-11:localhost:8800 62 | # 0.067986: Response to empty body 63 | # { 64 | # flows => [{ stages => ["m.login.password"] }], 65 | # params => {}, 66 | # session => "YLsIpaqBsQnFEvyukVgoLmPW", 67 | # } 68 | # 0.075230: Response to wrong password 69 | # { 70 | # completed => [], 71 | # errcode => "M_FORBIDDEN", 72 | # error => "Invalid password", 73 | # flows => [{ stages => ["m.login.password"] }], 74 | # params => {}, 75 | # session => "aMvuULWahaoHBbyXHFBoczOu", 76 | # } 77 | not ok 44 DELETE /device/{deviceId} requires UI auth user to match device owner 78 | # Started: 2019-07-31 19:41:12.730 79 | # Ended: 2019-07-31 19:41:12.803 80 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/devices/login_device?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYWNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTEyOmxvY2FsaG9zdDo4ODAwCjAwMTZjaWQgdHlwZSA9IGFjY2VzcwowMDIxY2lkIG5vbmNlID0gSlFQJjp2T0R6I3RPZER3dQowMDJmc2lnbmF0dXJlIKMb-dZYNYl3PfDCyN7x0xZZRqDRAZBt39r86Qo9EBKuCg ) from DELETE https://localhost:8800/_matrix/client/r0/devices/login_device?... 81 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 82 | # 0.035620: Registered new user @anon-20190731_194105-12:localhost:8800 83 | # 0.038542: Registered new user @anon-20190731_194105-13:localhost:8800 84 | ok 45 DELETE /device/{deviceId} with no body gives a 401 85 | ok 46 GET /presence/:user_id/status fetches initial status 86 | ok 47 PUT /presence/:user_id/status updates my presence 87 | ok 48 POST /createRoom makes a public room 88 | ok 49 POST /createRoom makes a private room 89 | ok 50 POST /createRoom makes a private room with invites 90 | ok 51 POST /createRoom makes a room with a name 91 | ok 52 POST /createRoom makes a room with a topic 92 | ok 53 Can /sync newly created room 93 | ok 54 POST /createRoom creates a room with the given version 94 | ok 55 POST /createRoom rejects attempts to create rooms with numeric versions 95 | ok 56 POST /createRoom rejects attempts to create rooms with unknown versions 96 | ok 57 POST /createRoom ignores attempts to set the room version via creation_content 97 | ok 58 GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership 98 | ok 59 GET /rooms/:room_id/state/m.room.member/:user_id?format=event fetches my membership event 99 | ok 60 GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels 100 | ok 61 GET /rooms/:room_id/joined_members fetches my membership 101 | ok 62 GET /rooms/:room_id/initialSync fetches initial sync state 102 | ok 63 GET /publicRooms lists newly-created room 103 | ok 64 GET /directory/room/:room_alias yields room ID 104 | ok 65 GET /joined_rooms lists newly-created room 105 | ok 66 POST /rooms/:room_id/state/m.room.name sets name 106 | ok 67 GET /rooms/:room_id/state/m.room.name gets name 107 | ok 68 POST /rooms/:room_id/state/m.room.topic sets topic 108 | ok 69 GET /rooms/:room_id/state/m.room.topic gets topic 109 | ok 70 GET /rooms/:room_id/state fetches entire room state 110 | ok 71 POST /createRoom with creation content 111 | ok 72 PUT /directory/room/:room_alias creates alias 112 | ok 73 POST /rooms/:room_id/join can join a room 113 | ok 74 POST /join/:room_alias can join a room 114 | ok 75 POST /join/:room_id can join a room 115 | ok 76 POST /join/:room_id can join a room with custom content 116 | ok 77 POST /join/:room_alias can join a room with custom content 117 | ok 78 POST /rooms/:room_id/leave can leave a room 118 | ok 79 POST /rooms/:room_id/invite can send an invite 119 | ok 80 POST /rooms/:room_id/ban can ban a user 120 | ok 81 POST /rooms/:room_id/send/:event_type sends a message 121 | ok 82 PUT /rooms/:room_id/send/:event_type/:txn_id sends a message 122 | ok 83 PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id 123 | ok 84 GET /rooms/:room_id/messages returns a message 124 | ok 85 GET /rooms/:room_id/messages lazy loads members correctly 125 | ok 86 PUT /rooms/:room_id/typing/:user_id sets typing notification 126 | ok 87 GET /rooms/:room_id/state/m.room.power_levels can fetch levels 127 | ok 88 PUT /rooms/:room_id/state/m.room.power_levels can set levels 128 | ok 89 PUT power_levels should not explode if the old power levels were empty 129 | ok 90 Both GET and PUT work 130 | ok 91 POST /rooms/:room_id/receipt can create receipts 131 | ok 92 POST /rooms/:room_id/read_markers can create read marker 132 | ok 93 POST /media/v1/upload can create an upload 133 | ok 94 GET /media/v1/download can fetch the value again 134 | ok 95 GET /capabilities is present and well formed for registered user 135 | ok 96 GET /r0/capabilities is not public 136 | ok 1 Got recaptcha verify request (Register with a recaptcha) 137 | ok 2 Passed captcha validation (Register with a recaptcha) 138 | 1..2 139 | ok 97 Register with a recaptcha (2 subtests) 140 | ok 98 registration is idempotent, without username specified 141 | ok 99 registration is idempotent, with username specified 142 | ok 100 registration remembers parameters 143 | ok 101 registration accepts non-ascii passwords 144 | ok 102 registration with inhibit_login inhibits login 145 | ok 103 User signups are forbidden from starting with '_' 146 | ok 104 Can login with 3pid and password using m.login.password 147 | ok 105 login types include SSO 148 | ok 106 /login/cas/redirect redirects if the old m.login.cas login type is listed 149 | ok 107 Can login with new user via CAS 150 | not ok 108 Can logout current device 151 | # Started: 2019-07-31 19:41:20.171 152 | # Ended: 2019-07-31 19:41:20.221 153 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/logout?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYWNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTQ4OmxvY2FsaG9zdDo4ODAwCjAwMTZjaWQgdHlwZSA9IGFjY2VzcwowMDIxY2lkIG5vbmNlID0gTiZYNmRQJmgzWFBfQkd3NQowMDJmc2lnbmF0dXJlINHsHnb5BEozvwyLcitvxYSR6I1i-wPq09qWHv8b0itbCg ) from POST https://localhost:8800/_matrix/client/r0/logout?... 154 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 155 | # 0.021577: Registered new user @anon-20190731_194105-48:localhost:8800 156 | # 0.041590: /devices response (1): 157 | # { 158 | # devices => [ 159 | # { 160 | # device_id => "HWXSOAPOUR", 161 | # display_name => undef, 162 | # last_seen_ip => undef, 163 | # last_seen_ts => undef, 164 | # user_id => "\@anon-20190731_194105-48:localhost:8800", 165 | # }, 166 | # { 167 | # device_id => "BRGVNRVUPB", 168 | # display_name => undef, 169 | # last_seen_ip => undef, 170 | # last_seen_ts => undef, 171 | # user_id => "\@anon-20190731_194105-48:localhost:8800", 172 | # }, 173 | # ], 174 | # } 175 | not ok 109 Can logout all devices 176 | # Started: 2019-07-31 19:41:20.221 177 | # Ended: 2019-07-31 19:41:20.264 178 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/logout/all?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYWNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTQ5OmxvY2FsaG9zdDo4ODAwCjAwMTZjaWQgdHlwZSA9IGFjY2VzcwowMDIxY2lkIG5vbmNlID0gVXlrQiM0cjRyWC4uVUhGMAowMDJmc2lnbmF0dXJlIKF3x1tXC8rP6SPwtxofi9tbAcWllehcPzOHwBGRIGAyCg ) from POST https://localhost:8800/_matrix/client/r0/logout/all?... 179 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 180 | # 0.021659: Registered new user @anon-20190731_194105-49:localhost:8800 181 | ok 110 Request to logout with invalid an access token is rejected 182 | ok 111 Request to logout without an access token is rejected 183 | ok 112 After changing password, can't log in with old password 184 | ok 113 After changing password, can log in with new password 185 | ok 114 After changing password, existing session still works 186 | not ok 115 After changing password, a different session no longer works 187 | # Started: 2019-07-31 19:41:20.492 188 | # Ended: 2019-07-31 19:41:20.571 189 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/account/password?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYWNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTUzOmxvY2FsaG9zdDo4ODAwCjAwMTZjaWQgdHlwZSA9IGFjY2VzcwowMDIxY2lkIG5vbmNlID0gR1VEJixrT2xnO0h3cy1sbwowMDJmc2lnbmF0dXJlIOuGtDRvQvgcD4O6l90Pt-rtushqdjb1uTI7ox6ZGu1SCg ) from POST https://localhost:8800/_matrix/client/r0/account/password?... 190 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 191 | # 0.022931: Registered new user @anon-20190731_194105-53:localhost:8800 192 | not ok 116 Pushers created with a different access token are deleted on password change 193 | # Started: 2019-07-31 19:41:20.571 194 | # Ended: 2019-07-31 19:41:20.637 195 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/account/password?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYWNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTU0OmxvY2FsaG9zdDo4ODAwCjAwMTZjaWQgdHlwZSA9IGFjY2VzcwowMDIxY2lkIG5vbmNlID0gWmE6UHFrI25oOlZDTzpVSwowMDJmc2lnbmF0dXJlIIBbptXk2dhRPTtezOXgLzarHK6nAjoTIcD-0FZ4pFzYCg ) from POST https://localhost:8800/_matrix/client/r0/account/password?... 196 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 197 | # 0.022429: Registered new user @anon-20190731_194105-54:localhost:8800 198 | ok 117 Pushers created with a the same access token are not deleted on password change 199 | not ok 118 Can deactivate account 200 | # Started: 2019-07-31 19:41:20.699 201 | # Ended: 2019-07-31 19:41:20.736 202 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/account/deactivate?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYWNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTU2OmxvY2FsaG9zdDo4ODAwCjAwMTZjaWQgdHlwZSA9IGFjY2VzcwowMDIxY2lkIG5vbmNlID0gdzdsdGJ-Njo6cWg1cGhJaQowMDJmc2lnbmF0dXJlILkWRf05KSGJG6sYCBE6MIafNGg4XcBqr69J1qk5Ca4UCg ) from POST https://localhost:8800/_matrix/client/r0/account/deactivate?... 203 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 204 | # 0.022338: Registered new user @anon-20190731_194105-56:localhost:8800 205 | ok 119 Can't deactivate account with wrong password 206 | not ok 120 After deactivating account, can't log in with password 207 | # Started: 2019-07-31 19:41:20.769 208 | # Ended: 2019-07-31 19:41:20.805 209 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/account/deactivate?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYWNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTU4OmxvY2FsaG9zdDo4ODAwCjAwMTZjaWQgdHlwZSA9IGFjY2VzcwowMDIxY2lkIG5vbmNlID0gdE86VlE4XjExQE5YQjkxIwowMDJmc2lnbmF0dXJlIOXb-ojaU3s-hXwyxG1L6llXJ_y0IfsIWwvi_DwXMNyuCg ) from POST https://localhost:8800/_matrix/client/r0/account/deactivate?... 210 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 211 | # 0.022742: Registered new user @anon-20190731_194105-58:localhost:8800 212 | ok 121 initialSync sees my presence status 213 | ok 122 Presence change reports an event to myself 214 | ok 123 Friends presence changes reports events # skip lack of can_invite_presence 215 | ok 124 Room creation reports m.room.create to myself 216 | ok 125 Room creation reports m.room.member to myself 217 | ok 126 Setting room topic reports m.room.topic to myself 218 | ok 127 Global initialSync 219 | ok 128 Global initialSync with limit=0 gives no messages 220 | ok 129 Room initialSync 221 | ok 130 Room initialSync with limit=0 gives no messages 222 | ok 131 Setting state twice is idempotent 223 | ok 132 Joining room twice is idempotent 224 | ok 133 New room members see their own join event 225 | ok 134 New room members see existing users' presence in room initialSync 226 | ok 135 Existing members see new members' join events 227 | ok 136 Existing members see new members' presence 228 | ok 137 All room members see all room members' presence in global initialSync 229 | # Starting Homeserver using SyTest::HomeserverFactory::Synapse=HASH(0x562459254088) 230 | # Starting server-1 231 | # Clearing Pg database pg2 on 'localhost' 232 | #** DBI connect('dbname=sytest_template;host=localhost','postgres',...) failed: FATAL: database "sytest_template" does not exist at lib/SyTest/Homeserver.pm line 371. 233 | # Generating config for port 8829 234 | # Creating config for server 1 with command /venv/bin/python -m coverage run --rcfile=/src/.coveragerc -m synapse.app.homeserver --config-path /work/server-1/config.yaml --server-name localhost:8829 --generate-config --report-stats=no 235 | Config file '/work/server-1/config.yaml' already exists. Generating any missing config files. 236 | Generating signing key file /work/server-1/localhost:8829.signing.key 237 | # Starting server 1 for port 8829 with command /venv/bin/python -m coverage run --rcfile=/src/.coveragerc -m synapse.app.homeserver --config-path /work/server-1/config.yaml --server-name localhost:8829 238 | # Connecting to server 8829 239 | # Connected to server 8829 240 | # Started server-1 241 | ok 138 Remote users can join room by alias 242 | ok 139 New room members see their own join event 243 | ok 140 New room members see existing members' presence in room initialSync 244 | ok 141 Existing members see new members' join events 245 | ok 142 Existing members see new member's presence 246 | ok 143 New room members see first user's profile information in global initialSync 247 | ok 144 New room members see first user's profile information in per-room initialSync 248 | ok 145 Remote users may not join unfederated rooms 249 | ok 146 Local room members see posted message events 250 | ok 147 Fetching eventstream a second time doesn't yield the message again 251 | ok 148 Local non-members don't see posted message events 252 | ok 149 Local room members can get room messages 253 | ok 150 (expected fail) Remote room members also see posted message events # TODO passed but expected fail 254 | ok 151 Remote room members can get room messages 255 | ok 152 Message history can be paginated 256 | ok 153 Message history can be paginated over federation 257 | ok 154 Room aliases can contain Unicode 258 | ok 155 Remote room alias queries can handle Unicode 259 | ok 1 m.room.canonical_alias accepts present aliases (Canonical alias can be set) 260 | ok 2 m.room.canonical_alias rejects missing aliases (Canonical alias can be set) 261 | 1..2 262 | ok 156 Canonical alias can be set (2 subtests) 263 | ok 157 Creators can delete alias 264 | ok 158 Deleting a non-existent alias should return a 404 265 | ok 159 Users can't delete other's aliases 266 | ok 160 Can delete canonical alias 267 | ok 161 Alias creators can delete alias with no ops 268 | ok 162 Alias creators can delete canonical alias with no ops 269 | ok 1 Sent invite (Can invite users to invite-only rooms) 270 | ok 2 Joined room (Can invite users to invite-only rooms) 271 | 1..2 272 | ok 163 Can invite users to invite-only rooms (2 subtests) 273 | ok 164 Uninvited users cannot join the room 274 | ok 165 Invited user can reject invite 275 | ok 166 Invited user can reject invite over federation 276 | ok 167 Invited user can reject invite over federation several times 277 | ok 168 Invited user can reject invite for empty room 278 | ok 169 Invited user can reject invite over federation for empty room 279 | ok 170 Invited user can reject local invite after originator leaves 280 | ok 171 Invited user can see room metadata 281 | ok 172 Remote invited user can see room metadata 282 | ok 173 Users cannot invite themselves to a room 283 | ok 174 Users cannot invite a user that is already in the room 284 | ok 175 Banned user is kicked and may not rejoin until unbanned 285 | ok 176 Remote banned user is kicked and may not rejoin until unbanned 286 | ok 1 Fails at powerlevel 0 ('ban' event respects room powerlevel) 287 | ok 2 Succeeds at powerlevel 100 ('ban' event respects room powerlevel) 288 | 1..2 289 | ok 177 'ban' event respects room powerlevel (2 subtests) 290 | ok 1 Fails at powerlevel 0 (setting 'm.room.name' respects room powerlevel) 291 | ok 2 Succeeds at powerlevel 100 (setting 'm.room.name' respects room powerlevel) 292 | 1..2 293 | ok 178 setting 'm.room.name' respects room powerlevel (2 subtests) 294 | ok 1 Fails at powerlevel 0 (setting 'm.room.power_levels' respects room powerlevel) 295 | ok 2 Succeeds at powerlevel 100 (setting 'm.room.power_levels' respects room powerlevel) 296 | 1..2 297 | ok 179 setting 'm.room.power_levels' respects room powerlevel (2 subtests) 298 | ok 180 Unprivileged users can set m.room.topic if it only needs level 0 299 | ok 1 Succeeds at setting 25 (Users cannot set ban powerlevel higher than their own) 300 | ok 2 Fails at setting 75 (Users cannot set ban powerlevel higher than their own) 301 | 1..2 302 | ok 181 Users cannot set ban powerlevel higher than their own (2 subtests) 303 | ok 1 Succeeds at setting 25 (Users cannot set kick powerlevel higher than their own) 304 | ok 2 Fails at setting 75 (Users cannot set kick powerlevel higher than their own) 305 | 1..2 306 | ok 182 Users cannot set kick powerlevel higher than their own (2 subtests) 307 | ok 1 Succeeds at setting 25 (Users cannot set redact powerlevel higher than their own) 308 | ok 2 Fails at setting 75 (Users cannot set redact powerlevel higher than their own) 309 | 1..2 310 | ok 183 Users cannot set redact powerlevel higher than their own (2 subtests) 311 | ok 1 Created a room (Check that event streams started after a client joined a room work (SYT-1)) 312 | ok 2 Alice saw her message (Check that event streams started after a client joined a room work (SYT-1)) 313 | 1..2 314 | ok 184 Check that event streams started after a client joined a room work (SYT-1) (2 subtests) 315 | ok 185 Event stream catches up fully after many messages 316 | ok 186 POST /rooms/:room_id/redact/:event_id as power user redacts message 317 | ok 187 POST /rooms/:room_id/redact/:event_id as original message sender redacts message 318 | ok 188 POST /rooms/:room_id/redact/:event_id as random user does not redact message 319 | ok 189 A departed room is still included in /initialSync (SPEC-216) 320 | ok 190 Can get rooms/{roomId}/initialSync for a departed room (SPEC-216) 321 | ok 191 Can get rooms/{roomId}/state for a departed room (SPEC-216) 322 | ok 192 Can get rooms/{roomId}/members for a departed room (SPEC-216) 323 | ok 193 Can get rooms/{roomId}/messages for a departed room (SPEC-216) 324 | ok 194 Can get 'm.room.name' state for a departed room (SPEC-216) 325 | ok 195 Getting messages going forward is limited for a departed room (SPEC-216) 326 | ok 196 Can invite existing 3pid 327 | ok 197 Can invite existing 3pid with no ops 328 | ok 198 Can invite existing 3pid in createRoom 329 | ok 199 Can invite unbound 3pid 330 | ok 200 Can invite unbound 3pid over federation 331 | ok 201 Can invite unbound 3pid with no ops 332 | ok 202 Can invite unbound 3pid over federation with no ops 333 | ok 203 Can invite unbound 3pid over federation with users from both servers 334 | ok 204 Can accept unbound 3pid invite after inviter leaves 335 | ok 205 Can accept third party invite with /join 336 | ok 206 3pid invite join with wrong but valid signature are rejected 337 | ok 207 3pid invite join valid signature but revoked keys are rejected 338 | ok 208 3pid invite join valid signature but unreachable ID server are rejected 339 | ok 209 Guest user cannot call /events globally 340 | ok 210 Guest users can join guest_access rooms 341 | ok 211 Guest users can send messages to guest_access rooms if joined 342 | ok 212 Guest user calling /events doesn't tightloop 343 | ok 213 Guest users are kicked from guest_access rooms on revocation of guest_access 344 | ok 214 Guest user can set display names 345 | ok 215 Guest users are kicked from guest_access rooms on revocation of guest_access over federation 346 | ok 216 Guest user can upgrade to fully featured user 347 | ok 217 Guest user cannot upgrade other users 348 | ok 218 GET /publicRooms lists rooms 349 | ok 219 GET /publicRooms includes avatar URLs 350 | not ok 220 (expected fail) Guest users can accept invites to private rooms over federation # TODO expected fail 351 | # Started: 2019-07-31 19:42:04.658 352 | # Ended: 2019-07-31 19:42:05.049 353 | # HTTP Request failed ( 403 Forbidden https://localhost:8800/_matrix/client/r0/join/!SMWSqgDDbLjXdwwXNE:localhost:8829?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAyNWNpZCB1c2VyX2lkID0gQDExOmxvY2FsaG9zdDo4ODAwCjAwMTZjaWQgdHlwZSA9IGFjY2VzcwowMDIxY2lkIG5vbmNlID0gNjd4elhoSEdJVyxjODtrZwowMDE1Y2lkIGd1ZXN0ID0gdHJ1ZQowMDJmc2lnbmF0dXJlIFiubrZR_2oAB3CCJgEU20M6iQ3bA02oX2_4GOJA2CZ5Cg ) from POST https://localhost:8800/_matrix/client/r0/join/!SMWSqgDDbLjXdwwXNE:localhost:8829?... 354 | # {"errcode":"M_FORBIDDEN","error":"Guest access not allowed"} 355 | # 0.032304: Registered new user @anon-20190731_194105-167:localhost:8829 356 | # 0.375427: Invited user @11:localhost:8800 to !SMWSqgDDbLjXdwwXNE:localhost:8829 357 | # {} 358 | ok 221 Guest users denied access over federation if guest access prohibited 359 | ok 222 Room members can override their displayname on a room-specific basis 360 | ok 223 Room members can join a room with an overridden displayname 361 | ok 224 Users cannot kick users from a room they are not in 362 | ok 225 Users cannot kick users who have already left a room 363 | ok 226 Typing notification sent to local room members 364 | ok 227 Typing notifications also sent to remote room members 365 | ok 228 Typing can be explicitly stopped 366 | ok 1 First m.read receipt is available (Read receipts are visible to /initialSync) 367 | ok 2 Updated m.read receipt is available (Read receipts are visible to /initialSync) 368 | 1..2 369 | ok 229 Read receipts are visible to /initialSync (2 subtests) 370 | ok 230 Read receipts are sent as events 371 | ok 231 Receipts must be m.read 372 | ok 232 displayname updates affect room member events 373 | ok 233 avatar_url updates affect room member events 374 | ok 234 m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users 375 | ok 235 m.room.history_visibility == "shared" allows/forbids appropriately for Guest users 376 | ok 236 m.room.history_visibility == "invited" allows/forbids appropriately for Guest users 377 | ok 237 m.room.history_visibility == "joined" allows/forbids appropriately for Guest users 378 | ok 238 m.room.history_visibility == "default" allows/forbids appropriately for Guest users 379 | ok 239 Guest non-joined user cannot call /events on shared room 380 | ok 240 Guest non-joined user cannot call /events on invited room 381 | ok 241 Guest non-joined user cannot call /events on joined room 382 | ok 242 Guest non-joined user cannot call /events on default room 383 | ok 243 Guest non-joined user can call /events on world_readable room 384 | ok 244 Guest non-joined user doesn't get events before room made world_readable 385 | ok 245 Guest non-joined users can get state for world_readable rooms 386 | ok 246 Guest non-joined users can get individual state for world_readable rooms 387 | ok 247 Guest non-joined users cannot room initalSync for non-world_readable rooms 388 | ok 248 Guest non-joined users can room initialSync for world_readable rooms 389 | ok 249 Guest non-joined users can get individual state for world_readable rooms after leaving 390 | ok 250 Guest non-joined users cannot send messages to guest_access rooms if not joined 391 | ok 251 Guest users can sync from world_readable guest_access rooms if joined 392 | ok 252 Guest users can sync from shared guest_access rooms if joined 393 | ok 253 Guest users can sync from invited guest_access rooms if joined 394 | ok 254 Guest users can sync from joined guest_access rooms if joined 395 | ok 255 Guest users can sync from default guest_access rooms if joined 396 | ok 256 m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users 397 | ok 257 m.room.history_visibility == "shared" allows/forbids appropriately for Real users 398 | ok 258 m.room.history_visibility == "invited" allows/forbids appropriately for Real users 399 | ok 259 m.room.history_visibility == "joined" allows/forbids appropriately for Real users 400 | ok 260 m.room.history_visibility == "default" allows/forbids appropriately for Real users 401 | ok 261 Real non-joined user cannot call /events on shared room 402 | ok 262 Real non-joined user cannot call /events on invited room 403 | ok 263 Real non-joined user cannot call /events on joined room 404 | ok 264 Real non-joined user cannot call /events on default room 405 | ok 265 Real non-joined user can call /events on world_readable room 406 | ok 266 Real non-joined user doesn't get events before room made world_readable 407 | ok 267 Real non-joined users can get state for world_readable rooms 408 | ok 268 Real non-joined users can get individual state for world_readable rooms 409 | ok 269 Real non-joined users cannot room initalSync for non-world_readable rooms 410 | ok 270 Real non-joined users can room initialSync for world_readable rooms 411 | ok 271 Real non-joined users can get individual state for world_readable rooms after leaving 412 | ok 272 Real non-joined users cannot send messages to guest_access rooms if not joined 413 | ok 273 Real users can sync from world_readable guest_access rooms if joined 414 | ok 274 Real users can sync from shared guest_access rooms if joined 415 | ok 275 Real users can sync from invited guest_access rooms if joined 416 | ok 276 Real users can sync from joined guest_access rooms if joined 417 | ok 277 Real users can sync from default guest_access rooms if joined 418 | ok 278 Only see history_visibility changes on boundaries 419 | ok 279 Backfill works correctly with history visibility set to joined 420 | ok 280 Forgotten room messages cannot be paginated 421 | ok 281 Forgetting room does not show up in v2 /sync 422 | ok 282 Can forget room you've been kicked from 423 | ok 283 Can't forget room you're still in 424 | ok 284 Can re-join room if re-invited 425 | not ok 285 Only original members of the room can see messages from erased users 426 | # Started: 2019-07-31 19:42:34.686 427 | # Ended: 2019-07-31 19:42:35.323 428 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/account/deactivate?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYmNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTI2Mzpsb2NhbGhvc3Q6ODgwMAowMDE2Y2lkIHR5cGUgPSBhY2Nlc3MKMDAyMWNpZCBub25jZSA9IGZyZXlNMHpsdTVxRXdPSTQKMDAyZnNpZ25hdHVyZSDQizj2zGtwN5fEETF5hSOV0PS1ZKAF-HQ5N1bK09N07wo ) from POST https://localhost:8800/_matrix/client/r0/account/deactivate?... 429 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 430 | # 0.150166: Registered new user @anon-20190731_194105-265:localhost:8800 431 | # 0.153601: Registered new user @anon-20190731_194105-264:localhost:8800 432 | # 0.154056: Registered new user @anon-20190731_194105-263:localhost:8800 433 | # 0.420401: User @anon-20190731_194105-264:localhost:8800 joined room 434 | # { room_id => "!dCfjHuFrGfMXBGsrNT:localhost:8800" } 435 | # 0.560384: User @anon-20190731_194105-265:localhost:8800 joined room 436 | # { room_id => "!dCfjHuFrGfMXBGsrNT:localhost:8800" } 437 | # 0.622354: messages for joining user before erasure 438 | # [ 439 | # { 440 | # content => { 441 | # creator => "\@anon-20190731_194105-263:localhost:8800", 442 | # room_version => 4, 443 | # }, 444 | # event_id => "\$EawTzpfK4xX1xgdXQ8ZU30YwYlVNRe5xj69bY19PE7I", 445 | # origin_server_ts => JSON::number(1564602154850), 446 | # sender => "\@anon-20190731_194105-263:localhost:8800", 447 | # state_key => "", 448 | # type => "m.room.create", 449 | # unsigned => { age => JSON::number(455) }, 450 | # }, 451 | # { 452 | # content => { 453 | # avatar_url => undef, 454 | # displayname => "anon-20190731_194105-263", 455 | # membership => "join", 456 | # }, 457 | # event_id => "\$GsIY3V6X8glZf3DyvTcVseLIPYVULNJ7EGJ55xBcYo4", 458 | # origin_server_ts => JSON::number(1564602154887), 459 | # sender => "\@anon-20190731_194105-263:localhost:8800", 460 | # state_key => "\@anon-20190731_194105-263:localhost:8800", 461 | # type => "m.room.member", 462 | # unsigned => { age => JSON::number(418) }, 463 | # }, 464 | # { 465 | # content => { 466 | # ban => JSON::number(50), 467 | # events => { 468 | # "m.room.avatar" => JSON::number(50), 469 | # "m.room.canonical_alias" => JSON::number(50), 470 | # "m.room.history_visibility" => JSON::number(100), 471 | # "m.room.name" => JSON::number(50), 472 | # "m.room.power_levels" => JSON::number(100), 473 | # }, 474 | # events_default => JSON::number(0), 475 | # invite => JSON::number(0), 476 | # kick => JSON::number(50), 477 | # redact => JSON::number(50), 478 | # state_default => JSON::number(50), 479 | # users => { 480 | # "\@anon-20190731_194105-263:localhost:8800" => JSON::number(100), 481 | # }, 482 | # users_default => JSON::number(0), 483 | # }, 484 | # event_id => "\$oFc9ULKyGt8YabFi8co98Whyf_1Zyi-gu6K_d6cLUy4", 485 | # origin_server_ts => JSON::number(1564602154931), 486 | # sender => "\@anon-20190731_194105-263:localhost:8800", 487 | # state_key => "", 488 | # type => "m.room.power_levels", 489 | # unsigned => { age => JSON::number(374) }, 490 | # }, 491 | # { 492 | # content => { join_rule => "public" }, 493 | # event_id => "\$W4hhxWQRVFRnzP-uGr81i2PJbn72WtgFYNuZQ7eZ7zg", 494 | # origin_server_ts => JSON::number(1564602154967), 495 | # sender => "\@anon-20190731_194105-263:localhost:8800", 496 | # state_key => "", 497 | # type => "m.room.join_rules", 498 | # unsigned => { age => JSON::number(338) }, 499 | # }, 500 | # { 501 | # content => { history_visibility => "shared" }, 502 | # event_id => "\$4FdgHdbmn9opD40xUvhIOisxqyMPi0FcyoFRaZ3m47w", 503 | # origin_server_ts => JSON::number(1564602155003), 504 | # sender => "\@anon-20190731_194105-263:localhost:8800", 505 | # state_key => "", 506 | # type => "m.room.history_visibility", 507 | # unsigned => { age => JSON::number(302) }, 508 | # }, 509 | # { 510 | # content => { 511 | # avatar_url => undef, 512 | # displayname => "anon-20190731_194105-264", 513 | # membership => "join", 514 | # }, 515 | # event_id => "\$Sj1JPCkogTKMXaPns_O_KLC0al8v5Xl0WBfIIfJUxl8", 516 | # origin_server_ts => JSON::number(1564602155078), 517 | # sender => "\@anon-20190731_194105-264:localhost:8800", 518 | # state_key => "\@anon-20190731_194105-264:localhost:8800", 519 | # type => "m.room.member", 520 | # unsigned => { age => JSON::number(227) }, 521 | # }, 522 | # { 523 | # content => { body => "body1", msgtype => "m.text" }, 524 | # event_id => "\$YWS5TeUnjPqzzZcOhqmG88NW46Ics9nJnWLX-Wc4xMY", 525 | # origin_server_ts => JSON::number(1564602155155), 526 | # sender => "\@anon-20190731_194105-263:localhost:8800", 527 | # type => "m.room.message", 528 | # unsigned => { age => JSON::number(150) }, 529 | # }, 530 | # { 531 | # content => { 532 | # avatar_url => undef, 533 | # displayname => "anon-20190731_194105-265", 534 | # membership => "join", 535 | # }, 536 | # event_id => "\$OyE1iBbFnBw6YEbjl2DI7DBSd41G4kQskgYB99sfOhQ", 537 | # origin_server_ts => JSON::number(1564602155217), 538 | # sender => "\@anon-20190731_194105-265:localhost:8800", 539 | # state_key => "\@anon-20190731_194105-265:localhost:8800", 540 | # type => "m.room.member", 541 | # unsigned => { age => JSON::number(88) }, 542 | # }, 543 | # ] 544 | ok 286 /joined_rooms returns only joined rooms 545 | ok 287 /joined_members return joined members 546 | ok 288 /context/ on joined room works 547 | ok 289 /context/ on non world readable room does not work 548 | ok 290 /context/ returns correct number of events 549 | ok 291 /context/ with lazy_load_members filter works 550 | ok 292 /event/ on joined room works 551 | ok 293 /event/ on non world readable room does not work 552 | ok 294 /event/ does not allow access to events before the user joined 553 | ok 295 Can get rooms/{roomId}/members 554 | ok 296 Can get rooms/{roomId}/members at a given point 555 | ok 297 Can filter rooms/{roomId}/members 556 | ok 298 /upgrade creates a new room 557 | ok 299 /upgrade should preserve room visibility for public rooms 558 | ok 300 /upgrade should preserve room visibility for private rooms 559 | ok 301 /upgrade copies the power levels to the new room 560 | ok 302 /upgrade copies important state to the new room 561 | ok 303 /upgrade copies ban events to the new room 562 | ok 304 /upgrade copies push rules to the new room 563 | ok 305 /upgrade moves aliases to the new room 564 | ok 306 /upgrade preserves direct room state 565 | ok 307 /upgrade preserves room federation ability 566 | ok 308 /upgrade restricts power levels in the old room 567 | ok 309 /upgrade restricts power levels in the old room when the old PLs are unusual 568 | ok 310 /upgrade to an unknown version is rejected 569 | ok 311 /upgrade is rejected if the user can't send state events 570 | ok 312 /upgrade of a bogus room fails gracefully 571 | ok 313 Name/topic keys are correct 572 | ok 314 Can get remote public room list 573 | ok 315 Can create filter 574 | ok 316 Can download filter 575 | ok 317 Can sync 576 | ok 318 Can sync a joined room 577 | ok 319 Full state sync includes joined rooms 578 | ok 320 Newly joined room is included in an incremental sync 579 | ok 321 Newly joined room has correct timeline in incremental sync 580 | ok 322 Newly joined room includes presence in incremental sync 581 | ok 323 Get presence for newly joined members in incremental sync 582 | ok 324 Can sync a room with a single message 583 | ok 325 Can sync a room with a message with a transaction id 584 | ok 326 A message sent after an initial sync appears in the timeline of an incremental sync. 585 | ok 327 A filtered timeline reaches its limit 586 | ok 328 Syncing a new room with a large timeline limit isn't limited 587 | ok 329 A full_state incremental update returns only recent timeline 588 | ok 330 A prev_batch token can be used in the v1 messages API 589 | ok 331 A next_batch token can be used in the v1 messages API 590 | ok 332 User sees their own presence in a sync 591 | ok 333 User is offline if they set_presence=offline in their sync 592 | ok 334 User sees updates to presence from other users in the incremental sync. 593 | ok 335 State is included in the timeline in the initial sync 594 | ok 336 State from remote users is included in the state in the initial sync 595 | ok 337 Changes to state are included in an incremental sync 596 | ok 338 Changes to state are included in an gapped incremental sync 597 | ok 339 State from remote users is included in the timeline in an incremental sync 598 | ok 340 A full_state incremental update returns all state 599 | ok 341 When user joins a room the state is included in the next sync 600 | ok 342 A change to displayname should not result in a full state sync 601 | ok 343 A change to displayname should appear in incremental /sync 602 | ok 344 When user joins a room the state is included in a gapped sync 603 | ok 345 When user joins and leaves a room in the same batch, the full state is still included in the next sync 604 | ok 346 Current state appears in timeline in private history 605 | ok 347 Current state appears in timeline in private history with many messages before 606 | ok 348 Current state appears in timeline in private history with many messages after 607 | ok 349 Rooms a user is invited to appear in an initial sync 608 | ok 350 Rooms a user is invited to appear in an incremental sync 609 | ok 351 Sync can be polled for updates 610 | ok 352 Sync is woken up for leaves 611 | ok 353 Left rooms appear in the leave section of sync 612 | ok 354 Newly left rooms appear in the leave section of incremental sync 613 | ok 355 We should see our own leave event, even if history_visibility is restricted (SYN-662) 614 | ok 356 We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462) 615 | ok 357 Newly left rooms appear in the leave section of gapped sync 616 | ok 358 Previously left rooms don't appear in the leave section of sync 617 | ok 359 Left rooms appear in the leave section of full state sync 618 | ok 360 Archived rooms only contain history from before the user left 619 | ok 361 Banned rooms appear in the leave section of sync 620 | ok 362 Newly banned rooms appear in the leave section of incremental sync 621 | ok 363 Newly banned rooms appear in the leave section of incremental sync 622 | ok 364 Typing events appear in initial sync 623 | ok 365 Typing events appear in incremental sync 624 | ok 366 Typing events appear in gapped sync 625 | ok 367 Read receipts appear in initial v2 /sync 626 | ok 368 New read receipts appear in incremental v2 /sync 627 | ok 369 Can pass a JSON filter as a query parameter 628 | ok 370 Can request federation format via the filter 629 | ok 371 Read markers appear in incremental v2 /sync 630 | ok 372 Read markers appear in initial v2 /sync 631 | ok 373 Read markers can be updated 632 | ok 374 Lazy loading parameters in the filter are strictly boolean 633 | ok 375 The only membership state included in an initial sync is for all the senders in the timeline 634 | ok 376 The only membership state included in an incremental sync is for senders in the timeline 635 | not ok 377 (expected fail) The only membership state included in a gapped incremental sync is for senders in the timeline # TODO expected fail 636 | # Started: 2019-07-31 19:43:28.431 637 | # Ended: 2019-07-31 19:43:30.083 638 | # Expected only 1 membership events at tests/10apidoc/09synced.pl line 327. 639 | # 0.051272: Registered new user @anon-20190731_194105-400:localhost:8800 640 | # 0.055371: Registered new user @anon-20190731_194105-398:localhost:8800 641 | # 0.057965: Registered new user @anon-20190731_194105-399:localhost:8800 642 | # 0.075529: Registered new user @anon-20190731_194105-401:localhost:8800 643 | # 0.560679: User @anon-20190731_194105-399:localhost:8800 joined room 644 | # { room_id => "!QphLfMjJgvLfakijzF:localhost:8800" } 645 | # 0.618187: User @anon-20190731_194105-400:localhost:8800 joined room 646 | # { room_id => "!QphLfMjJgvLfakijzF:localhost:8800" } 647 | # 0.975846: expected members: 648 | # [ 649 | # "\@anon-20190731_194105-398:localhost:8800", 650 | # "\@anon-20190731_194105-399:localhost:8800", 651 | # ] 652 | # 0.975993: state: 653 | # [ 654 | # { 655 | # content => { 656 | # ban => JSON::number(50), 657 | # events => { 658 | # "m.room.avatar" => JSON::number(50), 659 | # "m.room.canonical_alias" => JSON::number(50), 660 | # "m.room.history_visibility" => JSON::number(100), 661 | # "m.room.name" => JSON::number(50), 662 | # "m.room.power_levels" => JSON::number(100), 663 | # }, 664 | # events_default => JSON::number(0), 665 | # invite => JSON::number(0), 666 | # kick => JSON::number(50), 667 | # redact => JSON::number(50), 668 | # state_default => JSON::number(50), 669 | # users => { 670 | # "\@anon-20190731_194105-398:localhost:8800" => JSON::number(100), 671 | # }, 672 | # users_default => JSON::number(0), 673 | # }, 674 | # event_id => "\$Ffdnq8TS6tF9c2XAphVy1WNC260PWw7C5Ev7bmnvO4A", 675 | # origin_server_ts => JSON::number(1564602208768), 676 | # sender => "\@anon-20190731_194105-398:localhost:8800", 677 | # state_key => "", 678 | # type => "m.room.power_levels", 679 | # unsigned => { age => JSON::number(635) }, 680 | # }, 681 | # { 682 | # content => { 683 | # avatar_url => undef, 684 | # displayname => "anon-20190731_194105-399", 685 | # membership => "join", 686 | # }, 687 | # event_id => "\$1VYiMGywGnxgewD8c0IbvVea1aaeEUAnutp5WOTVk9E", 688 | # origin_server_ts => JSON::number(1564602208964), 689 | # sender => "\@anon-20190731_194105-399:localhost:8800", 690 | # state_key => "\@anon-20190731_194105-399:localhost:8800", 691 | # type => "m.room.member", 692 | # unsigned => { age => JSON::number(439) }, 693 | # }, 694 | # { 695 | # content => { join_rule => "public" }, 696 | # event_id => "\$67hbVMO9AZTOq5J6Srn0V0LDPpIJ5ipu4jOjD3BBKTg", 697 | # origin_server_ts => JSON::number(1564602208808), 698 | # sender => "\@anon-20190731_194105-398:localhost:8800", 699 | # state_key => "", 700 | # type => "m.room.join_rules", 701 | # unsigned => { age => JSON::number(595) }, 702 | # }, 703 | # { 704 | # content => { name => "A room name" }, 705 | # event_id => "\$oIcShwrugtyDcRRtvmfZcKGRZgP0VgVOaEB3H4-_ZJU", 706 | # origin_server_ts => JSON::number(1564602208916), 707 | # sender => "\@anon-20190731_194105-398:localhost:8800", 708 | # state_key => "", 709 | # type => "m.room.name", 710 | # unsigned => { age => JSON::number(487) }, 711 | # }, 712 | # { 713 | # content => { 714 | # avatar_url => undef, 715 | # displayname => "anon-20190731_194105-398", 716 | # membership => "join", 717 | # }, 718 | # event_id => "\$UFusl-eoNJ-YkMYMIwmLPlS2fdoDyrJnIhRaNvi0vjo", 719 | # origin_server_ts => JSON::number(1564602208588), 720 | # sender => "\@anon-20190731_194105-398:localhost:8800", 721 | # state_key => "\@anon-20190731_194105-398:localhost:8800", 722 | # type => "m.room.member", 723 | # unsigned => { age => JSON::number(815) }, 724 | # }, 725 | # { 726 | # content => { history_visibility => "shared" }, 727 | # event_id => "\$Y7QRnNxR2mDshfVfGoUlAkf8RcQJXIaMIaJhFJZXtsw", 728 | # origin_server_ts => JSON::number(1564602208845), 729 | # sender => "\@anon-20190731_194105-398:localhost:8800", 730 | # state_key => "", 731 | # type => "m.room.history_visibility", 732 | # unsigned => { age => JSON::number(558) }, 733 | # }, 734 | # { 735 | # content => { 736 | # creator => "\@anon-20190731_194105-398:localhost:8800", 737 | # room_version => 4, 738 | # }, 739 | # event_id => "\$sTXXvfz5RyRi_FqJVc33teH6Os1JFbD72RjEoaNduPc", 740 | # origin_server_ts => JSON::number(1564602208552), 741 | # sender => "\@anon-20190731_194105-398:localhost:8800", 742 | # state_key => "", 743 | # type => "m.room.create", 744 | # unsigned => { age => JSON::number(851) }, 745 | # }, 746 | # ] 747 | # 1.016837: User @anon-20190731_194105-401:localhost:8800 joined room 748 | # { room_id => "!QphLfMjJgvLfakijzF:localhost:8800" } 749 | # 1.650564: expected members: 750 | # ["\@anon-20190731_194105-401:localhost:8800"] 751 | # 1.650719: state: 752 | # [ 753 | # { 754 | # content => { 755 | # avatar_url => undef, 756 | # displayname => "anon-20190731_194105-400", 757 | # membership => "join", 758 | # }, 759 | # event_id => "\$y0qTvJVf34cHToXybrOC7ujI4i6kWL5MGF3eIt9JL2s", 760 | # origin_server_ts => JSON::number(1564602209019), 761 | # sender => "\@anon-20190731_194105-400:localhost:8800", 762 | # state_key => "\@anon-20190731_194105-400:localhost:8800", 763 | # type => "m.room.member", 764 | # unsigned => { age => JSON::number(1059) }, 765 | # }, 766 | # { 767 | # content => { 768 | # avatar_url => undef, 769 | # displayname => "anon-20190731_194105-401", 770 | # membership => "join", 771 | # }, 772 | # event_id => "\$SihiFKhYA20Y3B0fm3o6M_p7Vpnjm5W_g0T91cHJFpk", 773 | # origin_server_ts => JSON::number(1564602209419), 774 | # sender => "\@anon-20190731_194105-401:localhost:8800", 775 | # state_key => "\@anon-20190731_194105-401:localhost:8800", 776 | # type => "m.room.member", 777 | # unsigned => { age => JSON::number(659) }, 778 | # }, 779 | # ] 780 | ok 378 Gapped incremental syncs include all state changes 781 | ok 379 Old leaves are present in gapped incremental syncs 782 | ok 380 Leaves are present in non-gapped incremental syncs 783 | ok 381 Old members are included in gappy incr LL sync if they start speaking 784 | ok 382 Members from the gap are included in gappy incr LL sync 785 | ok 383 We don't send redundant membership state across incremental syncs by default 786 | ok 384 We do send redundant membership state across incremental syncs if asked 787 | ok 385 Unnamed room comes with a name summary 788 | ok 386 Named room comes with just joined member count summary 789 | ok 387 Room summary only has 5 heroes 790 | ok 388 Room summary counts change when membership changes 791 | ok 1 Can create room (User can create and send/receive messages in a room with version 1) 792 | ok 2 Can send/receive message in room (User can create and send/receive messages in a room with version 1) 793 | 1..2 794 | ok 389 User can create and send/receive messages in a room with version 1 (2 subtests) 795 | ok 390 local user can join room with version 1 796 | ok 391 User can invite local user to room with version 1 797 | ok 392 remote user can join room with version 1 798 | ok 393 User can invite remote user to room with version 1 799 | ok 394 Remote user can backfill in a room with version 1 800 | ok 395 Can reject invites over federation for rooms with version 1 801 | ok 396 Can receive redactions from regular users over federation in room version 1 802 | ok 1 Can create room (User can create and send/receive messages in a room with version 2) 803 | ok 2 Can send/receive message in room (User can create and send/receive messages in a room with version 2) 804 | 1..2 805 | ok 397 User can create and send/receive messages in a room with version 2 (2 subtests) 806 | ok 398 local user can join room with version 2 807 | ok 399 User can invite local user to room with version 2 808 | ok 400 remote user can join room with version 2 809 | ok 401 User can invite remote user to room with version 2 810 | ok 402 Remote user can backfill in a room with version 2 811 | ok 403 Can reject invites over federation for rooms with version 2 812 | ok 404 Can receive redactions from regular users over federation in room version 2 813 | ok 1 Can create room (User can create and send/receive messages in a room with version 3) 814 | ok 2 Can send/receive message in room (User can create and send/receive messages in a room with version 3) 815 | 1..2 816 | ok 405 User can create and send/receive messages in a room with version 3 (2 subtests) 817 | ok 406 local user can join room with version 3 818 | ok 407 User can invite local user to room with version 3 819 | ok 408 remote user can join room with version 3 820 | ok 409 User can invite remote user to room with version 3 821 | ok 410 Remote user can backfill in a room with version 3 822 | ok 411 Can reject invites over federation for rooms with version 3 823 | ok 412 Can receive redactions from regular users over federation in room version 3 824 | ok 1 Can create room (User can create and send/receive messages in a room with version 4) 825 | ok 2 Can send/receive message in room (User can create and send/receive messages in a room with version 4) 826 | 1..2 827 | ok 413 User can create and send/receive messages in a room with version 4 (2 subtests) 828 | ok 414 local user can join room with version 4 829 | ok 415 User can invite local user to room with version 4 830 | ok 416 remote user can join room with version 4 831 | ok 417 User can invite remote user to room with version 4 832 | ok 418 Remote user can backfill in a room with version 4 833 | ok 419 Can reject invites over federation for rooms with version 4 834 | ok 420 Can receive redactions from regular users over federation in room version 4 835 | ok 1 Can create room (User can create and send/receive messages in a room with version 5) 836 | ok 2 Can send/receive message in room (User can create and send/receive messages in a room with version 5) 837 | 1..2 838 | ok 421 User can create and send/receive messages in a room with version 5 (2 subtests) 839 | ok 422 local user can join room with version 5 840 | ok 423 User can invite local user to room with version 5 841 | ok 424 remote user can join room with version 5 842 | ok 425 User can invite remote user to room with version 5 843 | ok 426 Remote user can backfill in a room with version 5 844 | ok 427 Can reject invites over federation for rooms with version 5 845 | ok 428 Can receive redactions from regular users over federation in room version 5 846 | ok 429 Presence changes are reported to local room members 847 | ok 430 Presence changes are also reported to remote room members 848 | ok 431 Presence changes to UNAVAILABLE are reported to local room members 849 | ok 432 Presence changes to UNAVAILABLE are reported to remote room members 850 | not ok 433 (expected fail) Newly created users see their own presence in /initialSync (SYT-34) # TODO expected fail 851 | # Started: 2019-07-31 19:44:16.851 852 | # Ended: 2019-07-31 19:44:16.888 853 | # Expected to find my own presence at tests/40presence.pl line 140. 854 | # 0.022930: Registered new user @anon-20190731_194105-518:localhost:8800 855 | # 0.036454: initialSync response 856 | # { 857 | # account_data => [], 858 | # end => "s3111_606_14_10_9_4_1_551_1", 859 | # presence => [], 860 | # receipts => [], 861 | # rooms => [], 862 | # } 863 | ok 434 Can upload device keys 864 | not ok 435 (expected fail) Should reject keys claiming to belong to a different user # TODO expected fail 865 | # Started: 2019-07-31 19:44:16.928 866 | # Ended: 2019-07-31 19:44:16.938 867 | # Expected to receive an HTTP 4xx failure but it succeeded at ./run-tests.pl line 718. 868 | # 0.009487: Response 869 | # { one_time_key_counts => { my_algorithm => JSON::number(1) } } 870 | ok 436 Can query device keys using POST 871 | ok 437 Can query specific device keys using POST 872 | ok 438 query for user with no keys returns empty key dict 873 | ok 1 Uploaded one-time keys (Can claim one time key using POST) 874 | ok 2 Took one time key (Can claim one time key using POST) 875 | 1..2 876 | ok 439 Can claim one time key using POST (2 subtests) 877 | ok 1 Uploaded key (Can query remote device keys using POST) 878 | 1..1 879 | ok 440 Can query remote device keys using POST (1 subtests) 880 | ok 1 Uploaded one-time keys (Can claim remote one time key using POST) 881 | ok 2 Took one time key (Can claim remote one time key using POST) 882 | 1..2 883 | ok 441 Can claim remote one time key using POST (2 subtests) 884 | ok 442 Local device key changes appear in v2 /sync 885 | ok 443 Local new device changes appear in v2 /sync 886 | not ok 444 Local delete device changes appear in v2 /sync 887 | # Started: 2019-07-31 19:44:18.096 888 | # Ended: 2019-07-31 19:44:18.443 889 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/devices/HUVPZRLSEK?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYmNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTUzMTpsb2NhbGhvc3Q6ODgwMAowMDE2Y2lkIHR5cGUgPSBhY2Nlc3MKMDAyMWNpZCBub25jZSA9IG9zK3YyS2g4Mm1HcDA5OF8KMDAyZnNpZ25hdHVyZSAeCELIi3DIJ8MUCuFK1_Y9mbcX9casPuB-3ctAEw7Lxgo ) from DELETE https://localhost:8800/_matrix/client/r0/devices/HUVPZRLSEK?... 890 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 891 | # 0.038536: Registered new user @anon-20190731_194105-530:localhost:8800 892 | # 0.039585: Registered new user @anon-20190731_194105-531:localhost:8800 893 | # 0.285396: User @anon-20190731_194105-531:localhost:8800 joined room 894 | # { room_id => "!YNwTBVusLgskbrJFPo:localhost:8800" } 895 | ok 445 Local update device changes appear in v2 /sync 896 | ok 446 Can query remote device keys using POST after notification 897 | not ok 447 Device deletion propagates over federation 898 | # Started: 2019-07-31 19:44:19.414 899 | # Ended: 2019-07-31 19:44:20.209 900 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8829/_matrix/client/r0/devices/LXSWHLGCQV?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODI5CjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYmNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTUzNzpsb2NhbGhvc3Q6ODgyOQowMDE2Y2lkIHR5cGUgPSBhY2Nlc3MKMDAyMWNpZCBub25jZSA9IHVfTkpDXnhzOzBxMjFGeVIKMDAyZnNpZ25hdHVyZSChIqFk4OqSZt2X88qu-E1YlNr5RCfdg9RO7YU954Iasgo ) from DELETE https://localhost:8829/_matrix/client/r0/devices/LXSWHLGCQV?... 901 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 902 | # 0.034137: Registered new user @anon-20190731_194105-537:localhost:8829 903 | # 0.036567: Registered new user @anon-20190731_194105-536:localhost:8800 904 | # 0.310847: Invited user @anon-20190731_194105-537:localhost:8829 to !HHErjhHSZBetjnaxib:localhost:8800 905 | # {} 906 | # 0.621803: User @anon-20190731_194105-537:localhost:8829 joined room 907 | # { room_id => "!HHErjhHSZBetjnaxib:localhost:8800" } 908 | # 0.712739: sync_until_user_in_device_list: waiting for @anon-20190731_194105-537:localhost:8829 in changed 909 | # 0.744466: sync_until_user_in_device_list: body 910 | # { 911 | # account_data => { events => [] }, 912 | # device_lists => { 913 | # changed => ["\@anon-20190731_194105-537:localhost:8829"], 914 | # left => [], 915 | # }, 916 | # device_one_time_keys_count => {}, 917 | # groups => { invite => {}, join => {}, leave => {} }, 918 | # next_batch => "s3149_612_14_10_9_4_1_576_1", 919 | # presence => { events => [] }, 920 | # rooms => { invite => {}, join => {}, leave => {} }, 921 | # to_device => { events => [] }, 922 | # } 923 | # 0.745022: sync_until_user_in_device_list: found @anon-20190731_194105-537:localhost:8829 in changed 924 | # 0.757479: sync_until_user_in_device_list: waiting for @anon-20190731_194105-537:localhost:8829 in changed 925 | # 0.781059: sync_until_user_in_device_list: body 926 | # { 927 | # account_data => { events => [] }, 928 | # device_lists => { 929 | # changed => ["\@anon-20190731_194105-537:localhost:8829"], 930 | # left => [], 931 | # }, 932 | # device_one_time_keys_count => {}, 933 | # groups => { invite => {}, join => {}, leave => {} }, 934 | # next_batch => "s3149_612_14_10_9_4_1_577_1", 935 | # presence => { events => [] }, 936 | # rooms => { invite => {}, join => {}, leave => {} }, 937 | # to_device => { events => [] }, 938 | # } 939 | # 0.781726: sync_until_user_in_device_list: found @anon-20190731_194105-537:localhost:8829 in changed 940 | ok 448 If remote user leaves room, changes device and rejoins we see update in sync 941 | ok 449 If remote user leaves room we no longer receive device updates 942 | ok 450 Local device key changes appear in /keys/changes 943 | ok 451 New users appear in /keys/changes 944 | ok 452 If remote user leaves room, changes device and rejoins we see update in /keys/changes 945 | ok 453 Get left notifs in sync and /keys/changes when other user leaves 946 | ok 454 Get left notifs for other users in sync and /keys/changes when user leaves 947 | ok 455 If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes 948 | ok 456 Can create backup version 949 | ok 457 Can update backup version 950 | ok 458 Responds correctly when backup is empty 951 | ok 459 Can backup keys 952 | ok 460 Can update keys with better versions 953 | ok 461 Will not update keys with worse versions 954 | ok 462 Will not back up to an old backup version 955 | ok 463 Can delete backup 956 | ok 464 Deleted & recreated backups are empty 957 | ok 465 Can create more than 10 backup versions 958 | ok 466 Can add tag 959 | ok 467 Can remove tag 960 | ok 468 Can list tags for a room 961 | ok 469 Tags appear in the v1 /events stream 962 | ok 470 Tags appear in the v1 /initalSync 963 | ok 471 Tags appear in the v1 room initial sync 964 | ok 472 Tags appear in an initial v2 /sync 965 | ok 473 Newly updated tags appear in an incremental v2 /sync 966 | ok 474 Deleted tags appear in an incremental v2 /sync 967 | ok 475 /upgrade copies user tags to the new room 968 | ok 476 Can search for an event by body 969 | ok 477 Can get context around search results 970 | ok 478 Can back-paginate search results 971 | ok 479 Search works across an upgraded room and its predecessor 972 | ok 480 Can add account data 973 | ok 481 Can add account data to room 974 | ok 482 Can get account data without syncing 975 | ok 483 Can get room account data without syncing 976 | ok 484 Latest account data comes down in /initialSync 977 | ok 485 Latest account data comes down in room initialSync 978 | ok 486 Account data appears in v1 /events stream 979 | ok 487 Room account data appears in v1 /events stream 980 | ok 488 Latest account data appears in v2 /sync 981 | ok 489 New account data appears in incremental v2 /sync 982 | ok 490 Can generate a openid access_token that can be exchanged for information about a user 983 | ok 491 Invalid openid access tokens are rejected 984 | ok 492 Requests to userinfo without access tokens are rejected 985 | ok 493 Can send a message directly to a device using PUT /sendToDevice 986 | ok 494 Can recv a device message using /sync 987 | ok 495 Can recv device messages until they are acknowledged 988 | ok 496 Device messages with the same txn_id are deduplicated 989 | ok 497 Device messages wake up /sync 990 | ok 498 Can recv device messages over federation 991 | ok 499 Device messages over federation wake up /sync 992 | ok 500 Can send messages with a wildcard device id 993 | ok 501 Can send messages with a wildcard device id to two devices 994 | ok 502 Wildcard device messages wake up /sync 995 | ok 503 Wildcard device messages over federation wake up /sync 996 | ok 504 /whois 997 | ok 505 /purge_history 998 | ok 506 /purge_history by ts 999 | ok 507 Can backfill purged history 1000 | ok 1 Shutdown room returned success (Shutdown room) 1001 | ok 2 User cannot post in room (Shutdown room) 1002 | ok 3 User cannot rejoin room (Shutdown room) 1003 | ok 4 Remote users can't invite local users into room (Shutdown room) 1004 | ok 5 Aliases were repointed (Shutdown room) 1005 | ok 6 User was added to new room (Shutdown room) 1006 | ok 7 User cannot send into new room (Shutdown room) 1007 | 1..7 1008 | ok 508 Shutdown room (7 subtests) 1009 | ok 509 Ignore user in existing room 1010 | ok 510 Ignore invite in full sync 1011 | ok 511 Ignore invite in incremental sync 1012 | ok 512 Checking local federation server 1013 | ok 513 Federation key API allows unsigned requests for keys 1014 | ok 514 Federation key API can act as a notary server via a GET request 1015 | ok 515 Federation key API can act as a notary server via a POST request 1016 | ok 516 Key notary server should return an expired key if it can't find any others 1017 | ok 517 Key notary server must not overwrite a valid key with a spurious result from the origin server 1018 | ok 518 Outbound federation can query profile data 1019 | ok 519 Inbound federation can query profile data 1020 | ok 520 Outbound federation can query room alias directory 1021 | ok 521 Inbound federation can query room alias directory 1022 | ok 522 Outbound federation can send room-join requests 1023 | ok 523 Outbound federation passes make_join failures through to the client 1024 | #** SYN-490 detected; deploying workaround 1025 | ok 524 Inbound federation can receive room-join requests 1026 | ok 525 Inbound federation rejects remote attempts to join local users to rooms 1027 | ok 526 Inbound federation rejects remote attempts to kick local users to rooms 1028 | ok 527 Inbound federation rejects attempts to join v1 rooms from servers without v1 support 1029 | ok 528 Inbound federation rejects attempts to join v2 rooms from servers lacking version support 1030 | ok 529 Inbound federation rejects attempts to join v2 rooms from servers only supporting v1 1031 | ok 530 Inbound federation accepts attempts to join v2 rooms from servers with support 1032 | ok 531 Outbound federation correctly handles unsupported room versions 1033 | ok 532 A pair of servers can establish a join in a v2 room 1034 | ok 533 Outbound federation rejects send_join responses with no m.room.create event 1035 | ok 534 Outbound federation rejects m.room.create events with an unknown room version 1036 | ok 535 Outbound federation can send events 1037 | ok 536 Inbound federation can receive events 1038 | ok 537 Inbound federation can receive redacted events 1039 | ok 538 Inbound federation can return events 1040 | #** TODO: Unhandled incoming event of type 'm.room.message' at lib/SyTest/Federation/Server.pm line 462. 1041 | not ok 539 Inbound federation redacts events from erased users 1042 | # Started: 2019-07-31 19:44:49.878 1043 | # Ended: 2019-07-31 19:44:50.239 1044 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/account/deactivate?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYmNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTYzNDpsb2NhbGhvc3Q6ODgwMAowMDE2Y2lkIHR5cGUgPSBhY2Nlc3MKMDAyMWNpZCBub25jZSA9IHR6Xl9zZG5WNnRrVE1Ha3gKMDAyZnNpZ25hdHVyZSDKgHrR9JLV7lVljzvG5FNkBtSIAr50fV9Vg37S6bZ3dwo ) from POST https://localhost:8800/_matrix/client/r0/account/deactivate?... 1045 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 1046 | # 0.023827: Registered new user @anon-20190731_194105-634:localhost:8800 1047 | # 0.347576: Fetched event before erasure 1048 | # { 1049 | # origin => "localhost:8800", 1050 | # origin_server_ts => JSON::number(1564602290221), 1051 | # pdus => [ 1052 | # { 1053 | # auth_events => [ 1054 | # [ 1055 | # "\$1564602289347MAinE:localhost:8800", 1056 | # { sha256 => "SCNucT2CJRLagBkAOI1YUNAst94ujxmV2RftNv91lg0" }, 1057 | # ], 1058 | # [ 1059 | # "\$1564602289348objZO:localhost:8800", 1060 | # { sha256 => "2oWoKVFbUhetRRUILtSMjrnB97zCnBqqI4jG5A4sq60" }, 1061 | # ], 1062 | # [ 1063 | # "\$1564602289349OJayc:localhost:8800", 1064 | # { sha256 => "Jkc/SCSd7ABUpTkKmkX2gtXSrNtrvwWaudfCv4ZneOc" }, 1065 | # ], 1066 | # ], 1067 | # content => { body => "body1", msgtype => "m.text" }, 1068 | # depth => JSON::number(7), 1069 | # event_id => "\$1564602290353cAuGt:localhost:8800", 1070 | # hashes => { sha256 => "UOE4n9e+eDH3jEpZtSmFohpVQxCM4X72jZtJvKGVcjw" }, 1071 | # origin => "localhost:8800", 1072 | # origin_server_ts => JSON::number(1564602290180), 1073 | # prev_events => [ 1074 | # [ 1075 | # "\$18:localhost:37429", 1076 | # { sha256 => "ikAE36M1b+/ILQT+ZiBX/mHjfu+cqirNFzd5zpc+iyc" }, 1077 | # ], 1078 | # ], 1079 | # prev_state => [], 1080 | # room_id => "!wZvvAsAETTvjVBBjcY:localhost:8800", 1081 | # sender => "\@anon-20190731_194105-634:localhost:8800", 1082 | # signatures => { 1083 | # "localhost:8800" => { 1084 | # "ed25519:a_ULrx" => "Rv2Lvm9cPBI07fCix+Xsa3ZkqR9oLiENJvHDFDCdHrgSC7ttTL9iMBDNUZkZm7+LCnXuHmOTtHKAQ5NTCIxSAQ", 1085 | # }, 1086 | # }, 1087 | # type => "m.room.message", 1088 | # unsigned => { age => JSON::number(41) }, 1089 | # }, 1090 | # ], 1091 | # } 1092 | ok 540 Outbound federation can request missing events 1093 | ok 541 Inbound federation can return missing events for world_readable visibility 1094 | ok 542 Inbound federation can return missing events for shared visibility 1095 | ok 543 Inbound federation can return missing events for invite visibility 1096 | ok 544 Inbound federation can return missing events for joined visibility 1097 | ok 545 Outbound federation can backfill events 1098 | ok 546 Inbound federation can backfill events 1099 | ok 547 Backfill checks the events requested belong to the room 1100 | #** TODO: Respond to request to /_matrix/federation/v2/invite/!nVOfRumoRoqtUXvhqx:localhost:8800/$1564602294426NEwfx:localhost:8800 at lib/SyTest/Federation/Server.pm line 198. 1101 | ok 548 Outbound federation can send invites 1102 | ok 549 Inbound federation can receive invites via v1 API 1103 | ok 550 Inbound federation can receive invites via v2 API 1104 | ok 551 Inbound federation can receive invite and reject when remote replies with a 403 1105 | ok 552 Inbound federation can receive invite and reject when remote replies with a 500 1106 | ok 553 Inbound federation can receive invite and reject when is unreachable 1107 | ok 554 Inbound federation can get state for a room 1108 | ok 555 Inbound federation of state requires event_id as a mandatory paramater 1109 | ok 556 Inbound federation can get state_ids for a room 1110 | ok 557 Inbound federation of state_ids requires event_id as a mandatory paramater 1111 | ok 558 Federation rejects inbound events where the prev_events cannot be found 1112 | ok 559 Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state 1113 | ok 560 Federation handles empty auth_events in state_ids sanely 1114 | ok 561 Getting state checks the events requested belong to the room 1115 | ok 562 Getting state IDs checks the events requested belong to the room 1116 | ok 563 Should not be able to take over the room by pretending there is no PL event 1117 | ok 564 Inbound federation can get public room list 1118 | ok 565 Outbound federation sends receipts 1119 | #** TODO: Unhandled incoming event of type 'm.room.message' at lib/SyTest/Federation/Server.pm line 462. 1120 | ok 566 Inbound federation rejects receipts from wrong remote 1121 | ok 567 Inbound federation ignores redactions from invalid servers room > v3 1122 | ok 568 An event which redacts an event in a different room should be ignored 1123 | ok 569 An event which redacts itself should be ignored 1124 | ok 570 A pair of events which redact each other should be ignored 1125 | ok 571 Local device key changes get to remote servers 1126 | ok 572 Server correctly handles incoming m.device_list_update 1127 | #** TODO: Unhandled incoming event of type 'm.room.member' at lib/SyTest/Federation/Server.pm line 462. 1128 | ok 573 Local device key changes get to remote servers with correct prev_id 1129 | #** Cancelling unused user_keys_query await for at lib/SyTest/Federation/Server.pm line 334. 1130 | #** Cancelling unused user_devices await for @__ANON__-46:localhost:37429 at lib/SyTest/Federation/Server.pm line 334. 1131 | #** Cancelling unused user_keys_query await for at lib/SyTest/Federation/Server.pm line 334. 1132 | ok 574 Device list doesn't change if remote server is down 1133 | ok 575 Remote servers cannot set power levels in rooms without existing powerlevels 1134 | ok 576 Remote servers should reject attempts by non-creators to set the power levels 1135 | ok 577 Querying auth checks the events requested belong to the room 1136 | ok 578 Forward extremities remain so even after the next events are populated as outliers 1137 | ok 579 Server correctly handles transactions that break edu limits 1138 | #** TODO: Unhandled incoming event of type 'm.room.power_levels' at lib/SyTest/Federation/Server.pm line 462. 1139 | ok 580 Inbound federation correctly soft fails events 1140 | #** TODO: Unhandled incoming event of type 'm.room.power_levels' at lib/SyTest/Federation/Server.pm line 462. 1141 | ok 581 Inbound federation accepts a second soft-failed event 1142 | #** TODO: Unhandled incoming event of type 'm.room.power_levels' at lib/SyTest/Federation/Server.pm line 462. 1143 | ok 582 Inbound federation correctly handles soft failed events as extremities 1144 | ok 583 Can upload with Unicode file name 1145 | ok 584 Can download with Unicode file name locally 1146 | ok 585 Can download with Unicode file name over federation 1147 | # Starting proxy server 1148 | # Proxy now listening at port 39821 1149 | # connection to proxy server from 127.0.0.1:42164 1150 | # connected to localhost:8800 1151 | ok 586 Alternative server names do not cause a routing loop 1152 | ok 587 Can download specifying a different Unicode file name 1153 | ok 588 Can upload without a file name 1154 | ok 589 Can download without a file name locally 1155 | ok 590 Can download without a file name over federation 1156 | ok 591 Can upload with ASCII file name 1157 | ok 592 Can download file 'ascii' 1158 | ok 593 Can download file 'name with spaces' 1159 | ok 594 Can download file 'name;with;semicolons' 1160 | ok 595 Can download specifying a different ASCII file name 1161 | ok 596 Can send image in room message 1162 | ok 597 Can fetch images in room 1163 | ok 598 POSTed media can be thumbnailed 1164 | ok 599 Remote media can be thumbnailed 1165 | ok 1 URL was fetched (Test URL preview) 1166 | ok 2 Image was fetched (Test URL preview) 1167 | ok 3 Preview returned successfully (Test URL preview) 1168 | ok 4 Preview API returned expected values (Test URL preview) 1169 | 1..4 1170 | ok 600 Test URL preview (4 subtests) 1171 | ok 601 Can read configuration endpoint 1172 | ok 1 Quarantine returns success (Can quarantine media in rooms) 1173 | ok 2 404 on getting quarantined local media (Can quarantine media in rooms) 1174 | ok 3 404 on getting quarantined local thumbnails (Can quarantine media in rooms) 1175 | ok 4 404 on getting quarantined remote media (Can quarantine media in rooms) 1176 | ok 5 404 on getting quarantined remote thumbnails (Can quarantine media in rooms) 1177 | 1..5 1178 | ok 602 Can quarantine media in rooms (5 subtests) 1179 | ok 603 User appears in user directory 1180 | ok 604 User in private room doesn't appear in user directory 1181 | ok 1 User initially not in directory (User joining then leaving public room appears and dissappears from directory) 1182 | ok 2 User appears in directory after join (User joining then leaving public room appears and dissappears from directory) 1183 | ok 3 User not in directory after leaving room (User joining then leaving public room appears and dissappears from directory) 1184 | 1..3 1185 | ok 605 User joining then leaving public room appears and dissappears from directory (3 subtests) 1186 | ok 1 User initially not in directory (Users appear/disappear from directory when join_rules are changed) 1187 | ok 2 User appears in directory after join_rules set to public (Users appear/disappear from directory when join_rules are changed) 1188 | ok 3 User not in directory after join_rules set to private (Users appear/disappear from directory when join_rules are changed) 1189 | 1..3 1190 | ok 606 Users appear/disappear from directory when join_rules are changed (3 subtests) 1191 | ok 1 User initially not in directory (Users appear/disappear from directory when history_visibility are changed) 1192 | ok 2 User appears in directory after history_visibility set to public (Users appear/disappear from directory when history_visibility are changed) 1193 | ok 3 User not in directory after history_visibility set to private (Users appear/disappear from directory when history_visibility are changed) 1194 | 1..3 1195 | ok 607 Users appear/disappear from directory when history_visibility are changed (3 subtests) 1196 | ok 1 User initially not in directory (Users stay in directory when join_rules are changed but history_visibility is world_readable) 1197 | ok 2 User appears in directory after join_rules set to public (Users stay in directory when join_rules are changed but history_visibility is world_readable) 1198 | ok 3 User still in directory after join_rules set to invite (Users stay in directory when join_rules are changed but history_visibility is world_readable) 1199 | ok 4 User not in directory after history_visibility set to shared (Users stay in directory when join_rules are changed but history_visibility is world_readable) 1200 | 1..4 1201 | ok 608 Users stay in directory when join_rules are changed but history_visibility is world_readable (4 subtests) 1202 | ok 609 User in remote room doesn't appear in user directory after server left room 1203 | ok 610 User directory correctly update on display name change 1204 | ok 611 User in shared private room does appear in user directory 1205 | ok 612 User in shared private room does appear in user directory until leave 1206 | ok 613 User in dir while user still shares private rooms 1207 | ok 614 Create group 1208 | ok 615 Add group rooms 1209 | ok 616 Remove group rooms 1210 | ok 617 Get local group profile 1211 | ok 618 Get local group users 1212 | ok 619 Add/remove local group rooms 1213 | ok 620 Get local group summary 1214 | ok 621 Get remote group profile 1215 | ok 622 Get remote group users 1216 | ok 623 Add/remove remote group rooms 1217 | ok 624 Get remote group summary 1218 | ok 625 Add local group users 1219 | ok 626 Remove self from local group 1220 | ok 627 Remove other from local group 1221 | ok 628 Add remote group users 1222 | ok 629 Remove self from remote group 1223 | ok 630 Listing invited users of a remote group when not a member returns a 403 1224 | ok 631 Add group category 1225 | ok 632 Remove group category 1226 | ok 633 Get group categories 1227 | ok 634 Add group role 1228 | ok 635 Remove group role 1229 | ok 636 Get group roles 1230 | ok 637 Add room to group summary 1231 | ok 638 Adding room to group summary keeps room_id when fetching rooms in group 1232 | ok 639 Adding multiple rooms to group summary have correct order 1233 | ok 640 Remove room from group summary 1234 | ok 641 Add room to group summary with category 1235 | ok 642 Remove room from group summary with category 1236 | ok 643 Add user to group summary 1237 | ok 644 Adding multiple users to group summary have correct order 1238 | ok 645 Remove user from group summary 1239 | ok 646 Add user to group summary with role 1240 | ok 647 Remove user from group summary with role 1241 | ok 648 Local group invites come down sync 1242 | ok 649 Group creator sees group in sync 1243 | ok 650 Group creator sees group in initial sync 1244 | ok 651 Get/set local group publicity 1245 | ok 652 Bulk get group publicity 1246 | ok 653 Joinability comes down summary 1247 | ok 654 Set group joinable and join it 1248 | ok 655 Group is not joinable by default 1249 | ok 656 Group is joinable over federation 1250 | ok 657 Can bind 3PID via home server 1251 | ok 658 Can bind and unbind 3PID via homeserver 1252 | ok 659 Can unbind 3PID via homeserver when bound out of band 1253 | not ok 660 3PIDs are unbound after account deactivation 1254 | # Started: 2019-07-31 19:45:36.944 1255 | # Ended: 2019-07-31 19:45:37.011 1256 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/account/deactivate?access_token=MDAxY2xvY2F0aW9uIGxvY2FsaG9zdDo4ODAwCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzYmNpZCB1c2VyX2lkID0gQGFub24tMjAxOTA3MzFfMTk0MTA1LTgzNzpsb2NhbGhvc3Q6ODgwMAowMDE2Y2lkIHR5cGUgPSBhY2Nlc3MKMDAyMWNpZCBub25jZSA9IFJvbGRfSVgtOEZrTHN6dTIKMDAyZnNpZ25hdHVyZSA5PUEn9gvDN0QtHup9NubDSFLbSThuKj4nEPfjm77Bzgo ) from POST https://localhost:8800/_matrix/client/r0/account/deactivate?... 1257 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 1258 | # 0.025536: Registered new user @anon-20190731_194105-837:localhost:8800 1259 | ok 661 AS can create a user 1260 | ok 662 AS can create a user with an underscore 1261 | ok 663 AS can create a user with inhibit_login 1262 | ok 664 AS cannot create users outside its own namespace 1263 | ok 665 Regular users cannot register within the AS namespace 1264 | ok 666 AS can make room aliases 1265 | ok 667 Regular users cannot create room aliases within the AS namespace 1266 | ok 1 User joined room via AS (AS-ghosted users can use rooms via AS) 1267 | ok 2 User posted message via AS (AS-ghosted users can use rooms via AS) 1268 | 1..2 1269 | ok 668 AS-ghosted users can use rooms via AS (2 subtests) 1270 | ok 1 Ghost joined room themselves (AS-ghosted users can use rooms themselves) 1271 | ok 2 Ghost posted message themselves (AS-ghosted users can use rooms themselves) 1272 | ok 3 Creator received ghost's message (AS-ghosted users can use rooms themselves) 1273 | 1..3 1274 | ok 669 AS-ghosted users can use rooms themselves (3 subtests) 1275 | ok 670 Ghost user must register before joining room 1276 | #** Ignoring incoming AS event of type m.room.member 1277 | ok 671 AS can set avatar for ghosted users 1278 | ok 672 AS can set displayname for ghosted users 1279 | ok 673 AS can't set displayname for random users 1280 | ok 1 Sent invite (Inviting an AS-hosted user asks the AS server) 1281 | 1..1 1282 | ok 674 Inviting an AS-hosted user asks the AS server (1 subtests) 1283 | ok 1 Received AS request (Accesing an AS-hosted room alias asks the AS server) 1284 | ok 2 Created room alias mapping (Accesing an AS-hosted room alias asks the AS server) 1285 | 1..2 1286 | ok 675 Accesing an AS-hosted room alias asks the AS server (2 subtests) 1287 | ok 676 Events in rooms with AS-hosted room aliases are sent to AS server 1288 | #** Ignoring incoming AS event of type m.room.member 1289 | ok 677 AS user (not ghost) can join room without registering 1290 | #** Ignoring incoming AS event of type m.room.member 1291 | #** Ignoring incoming AS event of type m.room.member 1292 | ok 678 AS user (not ghost) can join room without registering, with user_id query param 1293 | #** Ignoring incoming AS event of type m.room.member 1294 | ok 679 HS provides query metadata 1295 | ok 680 HS can provide query metadata on a single protocol 1296 | ok 681 HS will proxy request for 3PU mapping 1297 | ok 682 HS will proxy request for 3PL mapping 1298 | ok 683 AS can publish rooms in their own list 1299 | ok 684 AS and main public room lists are separate 1300 | not ok 685 AS can deactivate a user 1301 | # Started: 2019-07-31 19:45:40.971 1302 | # Ended: 2019-07-31 19:45:41.000 1303 | # HTTP Request failed ( 500 Internal Server Error https://localhost:8800/_matrix/client/r0/account/deactivate?access_token=lFuOrdhx~K~lW%5EWfl%7B%40jyxDQxJHdnUeE&user_id=%40astest-4-20190731_194105%3Alocalhost%3A8800 ) from POST https://localhost:8800/_matrix/client/r0/account/deactivate?... 1304 | # {"errcode":"M_UNKNOWN","error":"Internal server error"} 1305 | ok 1 Bob received invite (Test that a message is pushed) 1306 | ok 2 Alice's pusher created (Test that a message is pushed) 1307 | ok 3 Message sent (Test that a message is pushed) 1308 | ok 4 Alice was pushed (Test that a message is pushed) 1309 | ok 5 Receipt sent (Test that a message is pushed) 1310 | ok 6 Zero badge push received (Test that a message is pushed) 1311 | 1..6 1312 | ok 686 Test that a message is pushed (6 subtests) 1313 | ok 687 Invites are pushed 1314 | ok 688 Rooms with aliases are correctly named in pushed 1315 | ok 689 Rooms with names are correctly named in pushed 1316 | ok 690 Rooms with canonical alias are correctly named in pushed 1317 | ok 691 Rooms with many users are correctly pushed 1318 | ok 692 Don't get pushed for rooms you've muted 1319 | ok 693 Can add global push rule for room 1320 | ok 694 Can add global push rule for sender 1321 | ok 695 Can add global push rule for content 1322 | ok 696 Can add global push rule for override 1323 | ok 697 Can add global push rule for underride 1324 | #** Use of strings with code points over 0xFF as arguments to bitwise xor (^) operator is deprecated at lib/SyTest/HTTPClient.pm line 151. 1325 | #** Use of strings with code points over 0xFF as arguments to bitwise xor (^) operator is deprecated at lib/SyTest/HTTPClient.pm line 151. 1326 | #** Use of strings with code points over 0xFF as arguments to bitwise xor (^) operator is deprecated at lib/SyTest/HTTPClient.pm line 151. 1327 | #** Use of strings with code points over 0xFF as arguments to bitwise xor (^) operator is deprecated at lib/SyTest/HTTPClient.pm line 151. 1328 | #** Use of strings with code points over 0xFF as arguments to bitwise xor (^) operator is deprecated at lib/SyTest/HTTPClient.pm line 151. 1329 | #** Use of strings with code points over 0xFF as arguments to bitwise xor (^) operator is deprecated at lib/SyTest/HTTPClient.pm line 151. 1330 | #** Use of strings with code points over 0xFF as arguments to bitwise xor (^) operator is deprecated at lib/SyTest/HTTPClient.pm line 151. 1331 | #** Use of strings with code points over 0xFF as arguments to bitwise xor (^) operator is deprecated at lib/SyTest/HTTPClient.pm line 151. 1332 | ok 698 Can add global push rule for content 1333 | ok 699 New rules appear before old rules by default 1334 | ok 700 Can add global push rule before an existing rule 1335 | ok 701 Can add global push rule after an existing rule 1336 | ok 702 Can delete a push rule 1337 | ok 703 Can disable a push rule 1338 | ok 704 Adding the same push rule twice is idempotent 1339 | ok 705 Messages that notify from another user increment unread notification count 1340 | ok 706 Messages that highlight from another user increment unread highlight count 1341 | ok 707 Can change the actions of default rules 1342 | ok 708 Changing the actions of an unknown default rule fails with 404 1343 | ok 709 Can change the actions of a user specified rule 1344 | ok 710 Changing the actions of an unknown rule fails with 404 1345 | ok 711 Can fetch a user's pushers 1346 | ok 712 Push rules come down in an initial /sync 1347 | ok 713 Adding a push rule wakes up an incremental /sync 1348 | ok 714 Disabling a push rule wakes up an incremental /sync 1349 | ok 715 Enabling a push rule wakes up an incremental /sync 1350 | ok 716 Setting actions for a push rule wakes up an incremental /sync 1351 | ok 717 Can enable/disable default rules 1352 | not ok 718 (expected fail) Enabling an unknown default rule fails with 404 # TODO expected fail 1353 | # Started: 2019-07-31 19:45:48.248 1354 | # Ended: 2019-07-31 19:45:48.279 1355 | # Expected to receive an HTTP 404 failure but it succeeded at ./run-tests.pl line 718. 1356 | # 0.022935: Registered new user @anon-20190731_194105-893:localhost:8800 1357 | # 0.030664: Response 1358 | # {} 1359 | ok 1 Alice's pusher 1 created (Test that rejected pushers are removed.) 1360 | ok 2 Alice's pusher 2 created (Test that rejected pushers are removed.) 1361 | ok 3 Message 1 Pushed (Test that rejected pushers are removed.) 1362 | ok 4 Message 2 Pushed (Test that rejected pushers are removed.) 1363 | 1..4 1364 | ok 719 Test that rejected pushers are removed. (4 subtests) 1365 | ok 720 Notifications can be viewed with GET /notifications 1366 | ok 721 Trying to add push rule with no scope fails with 400 1367 | ok 722 Trying to add push rule with invalid scope fails with 400 1368 | ok 723 Trying to add push rule with missing template fails with 400 1369 | ok 724 Trying to add push rule with missing rule_id fails with 400 1370 | ok 725 Trying to add push rule with empty rule_id fails with 400 1371 | ok 726 Trying to add push rule with invalid template fails with 400 1372 | ok 727 Trying to add push rule with rule_id with slashes fails with 400 1373 | ok 728 Trying to add push rule with override rule without conditions fails with 400 1374 | ok 729 Trying to add push rule with underride rule without conditions fails with 400 1375 | ok 730 Trying to add push rule with condition without kind fails with 400 1376 | ok 731 Trying to add push rule with content rule without pattern fails with 400 1377 | ok 732 Trying to add push rule with no actions fails with 400 1378 | ok 733 Trying to add push rule with invalid action fails with 400 1379 | ok 734 Trying to add push rule with invalid attr fails with 400 1380 | ok 735 Trying to add push rule with invalid value for enabled fails with 400 1381 | ok 736 Trying to get push rules with no trailing slash fails with 400 1382 | ok 737 Trying to get push rules with scope without trailing slash fails with 400 1383 | ok 738 Trying to get push rules with template without tailing slash fails with 400 1384 | ok 739 Trying to get push rules with unknown scope fails with 400 1385 | ok 740 Trying to get push rules with unknown template fails with 400 1386 | ok 741 Trying to get push rules with unknown attribute fails with 400 1387 | ok 742 Trying to get push rules with unknown rule_id fails with 404 1388 | ok 743 GET /initialSync with non-numeric 'limit' 1389 | ok 744 GET /events with non-numeric 'limit' 1390 | ok 745 GET /events with negative 'limit' 1391 | ok 746 GET /events with non-numeric 'timeout' 1392 | ok 747 Event size limits 1393 | ok 748 Check creating invalid filters returns 4xx 1394 | ok 1 Created a room (New federated private chats get full presence information (SYN-115)) 1395 | ok 2 Sent invite (New federated private chats get full presence information (SYN-115)) 1396 | ok 3 Received invite (New federated private chats get full presence information (SYN-115)) 1397 | ok 4 Joined room (New federated private chats get full presence information (SYN-115)) 1398 | ok 5 User @anon-20190731_194105-922:localhost:8800 received presence for @anon-20190731_194105-922:localhost:8800 (New federated private chats get full presence information (SYN-115)) 1399 | ok 6 User @anon-20190731_194105-922:localhost:8800 received presence for @anon-20190731_194105-923:localhost:8829 (New federated private chats get full presence information (SYN-115)) 1400 | ok 7 User @anon-20190731_194105-923:localhost:8829 received presence for @anon-20190731_194105-922:localhost:8800 (New federated private chats get full presence information (SYN-115)) 1401 | ok 8 User @anon-20190731_194105-923:localhost:8829 received presence for @anon-20190731_194105-923:localhost:8829 (New federated private chats get full presence information (SYN-115)) 1402 | ok 9 Both users see both users' presence (New federated private chats get full presence information (SYN-115)) 1403 | 1..9 1404 | ok 749 (expected fail) New federated private chats get full presence information (SYN-115) (9 subtests) # TODO passed but expected fail 1405 | ok 1 Created room (Left room members do not cause problems for presence) 1406 | ok 2 Left room (Left room members do not cause problems for presence) 1407 | 1..2 1408 | ok 750 Left room members do not cause problems for presence (2 subtests) 1409 | ok 1 Created room (Rooms can be created with an initial invite list (SYN-205)) 1410 | ok 2 Invitee received invite event (Rooms can be created with an initial invite list (SYN-205)) 1411 | 1..2 1412 | ok 751 Rooms can be created with an initial invite list (SYN-205) (2 subtests) 1413 | ok 1 Created room (Typing notifications don't leak) 1414 | ok 2 Members received notification (Typing notifications don't leak) 1415 | ok 3 Non-member did not receive it up to timeout (Typing notifications don't leak) 1416 | 1..3 1417 | ok 752 Typing notifications don't leak (3 subtests) 1418 | ok 1 Created room (Non-present room members cannot ban others) 1419 | ok 2 Set powerlevel (Non-present room members cannot ban others) 1420 | ok 3 Attempt to ban is rejected (Non-present room members cannot ban others) 1421 | 1..3 1422 | ok 753 Non-present room members cannot ban others (3 subtests) 1423 | ok 1 Set push rules for user (Getting push rules doesn't corrupt the cache SYN-390) 1424 | ok 2 Got push rules the first time (Getting push rules doesn't corrupt the cache SYN-390) 1425 | ok 3 Got push rules the second time (Getting push rules doesn't corrupt the cache SYN-390) 1426 | 1..3 1427 | ok 754 Getting push rules doesn't corrupt the cache SYN-390 (3 subtests) 1428 | ok 1 User A created a room (Test that we can be reinvited to a room we created) 1429 | ok 2 User A set the join rules to 'invite' (Test that we can be reinvited to a room we created) 1430 | ok 3 User A invited user B (Test that we can be reinvited to a room we created) 1431 | ok 4 User B received the invite from A (Test that we can be reinvited to a room we created) 1432 | ok 5 User B joined the room (Test that we can be reinvited to a room we created) 1433 | ok 6 User A set user B's power level to 100 (Test that we can be reinvited to a room we created) 1434 | ok 7 User A left the room (Test that we can be reinvited to a room we created) 1435 | ok 8 User B received the leave event (Test that we can be reinvited to a room we created) 1436 | ok 9 User B invited user A back to the room (Test that we can be reinvited to a room we created) 1437 | ok 10 User A received the invite from user B (Test that we can be reinvited to a room we created) 1438 | ok 11 User A joined the room (Test that we can be reinvited to a room we created) 1439 | 1..11 1440 | ok 755 Test that we can be reinvited to a room we created (11 subtests) 1441 | ok 1 User A created a room (Multiple calls to /sync should not cause 500 errors) 1442 | ok 2 Sent typing notification (Multiple calls to /sync should not cause 500 errors) 1443 | ok 3 Sent message (Multiple calls to /sync should not cause 500 errors) 1444 | ok 4 Updated read receipt (Multiple calls to /sync should not cause 500 errors) 1445 | ok 5 Completed first sync (Multiple calls to /sync should not cause 500 errors) 1446 | ok 6 Completed second sync (Multiple calls to /sync should not cause 500 errors) 1447 | 1..6 1448 | ok 756 Multiple calls to /sync should not cause 500 errors (6 subtests) 1449 | ok 757 Guest user can call /events on another world_readable room (SYN-606) 1450 | ok 758 Real user can call /events on another world_readable room (SYN-606) 1451 | ok 759 Events come down the correct room 1452 | # Killing 166 1453 | # Killing 206 1454 | 206 stopped 1455 | 166 stopped 1456 | 1..759 1457 | -------------------------------------------------------------------------------- /spec/sample_artifacts/xunit-no-file.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Failure/Error: fail('whatever') 4 | 5 | RuntimeError: 6 | whatever 7 | ./spec/features/test_things_spec.rb:5:in `block (3 levels) in <top (required)>' 8 | 9 | -------------------------------------------------------------------------------- /spec/sample_artifacts/xunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | (http://localhost:9001/assets/tests.js:2188:20) at runTest (http://localhost:9001/assets/test-support.js:15012:30) at Test.run (http://localhost:9001/assets/test-support.js:14998:6) at http://localhost:9001/assets/test-support.js:15210:12 at advanceTaskQueue (http://localhost:9001/assets/test-support.js:14611:6) at Object.advance (http://localhost:9001/assets/test-support.js:14592:4) 8 | ]]> 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | Error: Expected image to match or be a close match to snapshot but was 8.546910755148742% different from snapshot (6723 differing pixels) 19 | 20 | 21 | 22 | 23 | 24 | 25 | Error: expect(value).toMatchSnapshot() 26 | 27 | 28 | Received value does not match stored snapshot "Header matches snapshot 1". 29 | 30 | - Snapshot 31 | + Received 32 | 33 | @@ -3,11 +3,11 @@ 34 | > 35 | <nav 36 | class="startGroup" 37 | > 38 | <a 39 | - class="base anchor button booo homeLink" 40 | + class="base anchor button linkButton homeLink" 41 | href="/" 42 | > 43 | <span 44 | class="text" 45 | > 46 | at Object.toMatchRenderedSnapshot (/build/web/tools/jest/matchers/to_match_rendered_snapshot.ts:9:25) 47 | at Object.throwingMatcher [as toMatchRenderedSnapshot] (/build/web/node_modules/expect/build/index.js:316:33) 48 | at Object.it (/build/web/src/pages/editor/header/tests/header.tests.tsx:76:9) 49 | at Object.asyncFn (/build/web/node_modules/jest-jasmine2/build/jasmine_async.js:108:37) 50 | at resolve (/build/web/node_modules/jest-jasmine2/build/queue_runner.js:56:12) 51 | at new Promise (<anonymous>) 52 | at mapper (/build/web/node_modules/jest-jasmine2/build/queue_runner.js:43:19) 53 | at promise.then (/build/web/node_modules/jest-jasmine2/build/queue_runner.js:87:41) 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'bundler/setup' 4 | require 'simplecov' 5 | SimpleCov.start 6 | 7 | require 'test_summary_buildkite_plugin' 8 | 9 | Dir['./spec/support/**/*.rb'].each { |file| require file } 10 | 11 | RSpec.configure do |config| 12 | # Enable flags like --only-failures and --next-failure 13 | config.example_status_persistence_file_path = '.rspec_status' 14 | 15 | # Disable RSpec exposing methods globally on `Module` and `main` 16 | config.disable_monkey_patching! 17 | 18 | config.expect_with :rspec do |c| 19 | c.syntax = :expect 20 | end 21 | 22 | config.include Stubs 23 | end 24 | -------------------------------------------------------------------------------- /spec/support/stubs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Stubs 4 | def self.included(base) 5 | base.before do 6 | allow(TestSummaryBuildkitePlugin::Agent.instance).to receive(:run) do |*args| 7 | @agent_commands ||= [] 8 | @agent_commands << args 9 | end 10 | 11 | stub_const('TestSummaryBuildkitePlugin::Input::WORKDIR', 'spec/sample_artifacts') 12 | end 13 | end 14 | 15 | def agent_commands 16 | @agent_commands || [] 17 | end 18 | 19 | def agent_artifact_commands 20 | agent_commands.select { |x| x.first == 'artifact' } 21 | end 22 | 23 | def agent_annotate_commands 24 | agent_commands.select { |x| x.first == 'annotate' } 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /spec/test_summary_buildkite_plugin/failure_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | RSpec.describe TestSummaryBuildkitePlugin::Failure do 6 | describe TestSummaryBuildkitePlugin::Failure::Structured do 7 | let(:summary) { 'foo.rb: unicorns are\'t real' } 8 | let(:details) { 'This is a failure of great proportions' } 9 | let(:params) { { summary: summary, details: details } } 10 | 11 | describe 'strip_colors' do 12 | let(:details) { 'Failure/Error: \\e[0m\\e[32mit\\e[0m { \\e[32mexpect\\e[0m(url).to be_nil }' } 13 | 14 | subject(:failure) { described_class.new(params) } 15 | 16 | before { failure.strip_colors } 17 | 18 | it 'strips terminal color directives' do 19 | expect(failure.details).to eq('Failure/Error: it { expect(url).to be_nil }') 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/test_summary_buildkite_plugin/formatter_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | RSpec.describe TestSummaryBuildkitePlugin::Formatter do 6 | let(:show_first) { nil } 7 | let(:truncate) { nil } 8 | let(:input) { double(TestSummaryBuildkitePlugin::Input::Base, label: 'animals') } 9 | let(:failures) { [] } 10 | let(:options) { { type: type, show_first: show_first, truncate: truncate } } 11 | 12 | subject(:markdown) { described_class.create(options).markdown(input) } 13 | 14 | before do 15 | allow(input).to receive(:failures).and_return(failures) 16 | end 17 | 18 | describe 'details' do 19 | let(:type) { 'details' } 20 | 21 | context 'with no failures' do 22 | let(:failures) { [] } 23 | 24 | it 'returns empty markdown' do 25 | expect(markdown).to be_nil 26 | end 27 | end 28 | 29 | context 'with no details' do 30 | let(:failures) { [TestSummaryBuildkitePlugin::Failure::Unstructured.new('ponies are awesome')] } 31 | 32 | it 'includes the label' do 33 | expect(markdown).to include('animals') 34 | end 35 | 36 | it 'includes the summary' do 37 | expect(markdown).to include('ponies are awesome') 38 | end 39 | 40 | it 'has no
elements' do 41 | expect(markdown).not_to include(' element' do 72 | expect(markdown).to include('& amazing' } 85 | 86 | it 'escapes angle brackets' do 87 | expect(markdown).to include('<') 88 | end 89 | 90 | it 'escapes "&"' do 91 | expect(markdown).to include('&') 92 | end 93 | end 94 | 95 | context 'with html chars in description' do 96 | let(:details) { 'like, really awesome & amazing' } 97 | 98 | it 'escapes angle brackets' do 99 | expect(markdown).to include('<') 100 | end 101 | 102 | it 'escapes "&"' do 103 | expect(markdown).to include('&') 104 | end 105 | end 106 | 107 | context 'with blank lines in description' do 108 | let(:details) { "one\n\ntwo" } 109 | 110 | it 'adds nbsp so CommonMark does not terminate the block' do 111 | expect(markdown).to include("one\n \ntwo") 112 | end 113 | end 114 | 115 | context 'with multiple consecutive blank lines in description' do 116 | let(:details) { "one\n\n\n\ntwo" } 117 | 118 | it 'strips subsequent blank lines' do 119 | expect(markdown).to include("one\n \ntwo") 120 | end 121 | end 122 | end 123 | end 124 | 125 | describe 'summary' do 126 | let(:type) { 'summary' } 127 | 128 | context 'with no failures' do 129 | let(:failures) { [] } 130 | 131 | it 'returns empty markdown' do 132 | expect(markdown).to be_nil 133 | end 134 | end 135 | 136 | context 'with failures' do 137 | let(:failures) { [TestSummaryBuildkitePlugin::Failure::Unstructured.new('ponies are awesome')] } 138 | 139 | it 'includes the label' do 140 | expect(markdown).to include('animals') 141 | end 142 | 143 | it 'includes the summary' do 144 | expect(markdown).to include('ponies are awesome') 145 | end 146 | end 147 | end 148 | 149 | describe 'show_first' do 150 | let(:type) { 'summary' } 151 | let(:show_first) { 2 } 152 | let(:failures) do 153 | %w[dog cat pony horse unicorn].map { |x| TestSummaryBuildkitePlugin::Failure::Unstructured.new(x) } 154 | end 155 | let(:before_details) { markdown.split('\S+:\d+)' 148 | } } 149 | end 150 | 151 | it 'sets summary based on format' do 152 | expect(input.failures.map(&:summary)).to( 153 | include( 154 | './spec/features/sign_in_out_spec.rb:27: signing in and out sign_in reset password' 155 | ) 156 | ) 157 | end 158 | end 159 | end 160 | 161 | context 'disabling message' do 162 | let(:additional_options) do 163 | { message: false } 164 | end 165 | 166 | it 'ignores the message' do 167 | expect(input.failures.first.message).to be_nil 168 | end 169 | end 170 | 171 | context 'disabling details' do 172 | let(:additional_options) do 173 | { details: false } 174 | end 175 | 176 | it 'ignores the details' do 177 | expect(input.failures.first.details).to be_nil 178 | end 179 | end 180 | end 181 | 182 | describe 'with glob path' do 183 | let(:type) { 'junit' } 184 | let(:artifact_path) { 'rspec*' } 185 | 186 | it 'has all failures' do 187 | expect(input.failures.count).to eq(6) 188 | end 189 | 190 | it 'sorts failures' do 191 | expect(input.failures.first.summary).to include('./spec/features/sign_in_out_spec.rb') 192 | expect(input.failures.last.summary).to include('./spec/lib/url_whitelist_spec.rb') 193 | end 194 | end 195 | 196 | describe 'tap' do 197 | let(:type) { 'tap' } 198 | let(:artifact_path) { 'example.tap' } 199 | 200 | it { is_expected.to be_a(TestSummaryBuildkitePlugin::Input::Tap) } 201 | 202 | it 'has all failures' do 203 | expect(input.failures.count).to eq(2) 204 | end 205 | 206 | it 'failures have details' do 207 | expect(input.failures.first.details).to eq('message: \'timeout\' 208 | severity: fail') 209 | end 210 | 211 | it 'failures have summary' do 212 | expect(input.failures.first.summary).to eq('pinged quartz') 213 | end 214 | 215 | context 'version 12' do 216 | let(:artifact_path) { 'version_12.tap' } 217 | 218 | it { is_expected.to be_a(TestSummaryBuildkitePlugin::Input::Tap) } 219 | 220 | it 'has all failures' do 221 | expect(input.failures.count).to eq(14) 222 | end 223 | 224 | it 'ignores TODOs' do 225 | expect(input.failures.map(&:summary)).not_to include(include('expected fail')) 226 | end 227 | 228 | it 'failures have summary' do 229 | expect(input.failures.first.summary).to eq('3PIDs are unbound after account deactivation') 230 | end 231 | 232 | it 'failures have details' do 233 | expect(input.failures.first.details).to include('500 Internal Server Error') 234 | end 235 | end 236 | end 237 | 238 | describe 'checkstyle' do 239 | let(:type) { 'checkstyle' } 240 | let(:artifact_path) { 'checkstyle.xml' } 241 | 242 | it { is_expected.to be_a(TestSummaryBuildkitePlugin::Input::Checkstyle) } 243 | 244 | it 'has all failures' do 245 | expect(input.failures.count).to eq(3) 246 | end 247 | 248 | it 'failures have summary' do 249 | expect(input.failures[0].summary).to eq( 250 | '[error] src/main/java/io/timnew/sol/Sol.kt: Without column or line' 251 | ) 252 | expect(input.failures[1].summary).to eq( 253 | '[error] src/main/java/io/timnew/sol/Sol.kt:106:1: Needless blank line(s)' 254 | ) 255 | expect(input.failures[2].summary).to eq( 256 | '[error] src/main/java/io/timnew/sol/Sol.kt:109: Without column' 257 | ) 258 | end 259 | 260 | it 'failures have details' do 261 | expect(input.failures.first.details).to include 'no-consecutive-blank-lines' 262 | end 263 | end 264 | 265 | describe 'setting ascii encoding' do 266 | let(:type) { 'oneline' } 267 | let(:artifact_path) { 'eslint-00112233-0011-0011-0011-001122334455.txt' } 268 | let(:additional_options) { { encoding: 'ascii', fail_on_error: true } } 269 | 270 | it 'tries to parse as ascii' do 271 | expect { input.failures }.to raise_error('invalid byte sequence in US-ASCII') 272 | end 273 | end 274 | 275 | describe 'job_id' do 276 | let(:type) { 'oneline' } 277 | 278 | context 'with uuid in path' do 279 | let(:artifact_path) { 'eslint-00112233-0011-0011-0011-001122334455.txt' } 280 | 281 | it 'failures have job_id' do 282 | expect(input.failures.first.job_id).to eq('00112233-0011-0011-0011-001122334455') 283 | end 284 | end 285 | 286 | context 'without uuid in path' do 287 | let(:artifact_path) { 'rubocop.txt' } 288 | 289 | it 'failures have no job_id' do 290 | expect(input.failures.first.job_id).to be_nil 291 | end 292 | end 293 | 294 | context 'with custom job_id_regex' do 295 | let(:additional_options) { { job_id_regex: '(?eslint)' } } 296 | let(:artifact_path) { 'eslint-00112233-0011-0011-0011-001122334455.txt' } 297 | 298 | it 'uses the regex' do 299 | expect(input.failures.first.job_id).to eq('eslint') 300 | end 301 | end 302 | 303 | context 'with bad custom job_id_regex' do 304 | let(:additional_options) { { job_id_regex: '(eslint)', fail_on_error: true } } 305 | let(:artifact_path) { 'eslint-00112233-0011-0011-0011-001122334455.txt' } 306 | 307 | it 'gives helpful error' do 308 | expect { input.failures }.to raise_error(/must have a job_id named capture/) 309 | end 310 | end 311 | end 312 | 313 | describe 'agent exceptions' do 314 | let(:type) { 'oneline' } 315 | let(:artifact_path) { 'rubocop.txt' } 316 | 317 | before do 318 | allow(TestSummaryBuildkitePlugin::Agent).to receive(:run).and_raise( 319 | TestSummaryBuildkitePlugin::Agent::CommandFailed.new('life sucks') 320 | ) 321 | end 322 | 323 | context 'without fail_on_error' do 324 | it 'returns no failures' do 325 | expect(input.failures).to eq([]) 326 | end 327 | 328 | it 'logs the error' do 329 | expect { input.failures }.to output(/life sucks/).to_stdout 330 | end 331 | end 332 | 333 | context 'with fail_on_error' do 334 | let(:additional_options) { { fail_on_error: true } } 335 | 336 | it 'raises error' do 337 | expect { input.failures }.to raise_error(TestSummaryBuildkitePlugin::Agent::CommandFailed) 338 | end 339 | end 340 | end 341 | 342 | describe 'parsing exceptions' do 343 | let(:type) { 'oneline' } 344 | let(:artifact_path) { 'rubocop.txt' } 345 | 346 | before do 347 | allow(input).to receive(:file_contents_to_failures).and_raise('no good') 348 | end 349 | 350 | it 'handles the error' do 351 | expect { input.failures }.to output(/no good/).to_stdout 352 | expect(input.failures).to be_empty 353 | end 354 | 355 | describe 'with fail_on_error' do 356 | let(:additional_options) { { fail_on_error: true } } 357 | 358 | it 're-raises the error' do 359 | expect { input.failures }.to raise_error('no good') 360 | end 361 | end 362 | end 363 | end 364 | -------------------------------------------------------------------------------- /spec/test_summary_buildkite_plugin/runner_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | RSpec.describe TestSummaryBuildkitePlugin::Runner do 6 | let(:params) { { inputs: inputs } } 7 | let(:runner) { described_class.new(params) } 8 | 9 | subject(:run) { runner.run } 10 | 11 | context 'with no failures' do 12 | let(:inputs) do 13 | [ 14 | label: 'rspec', 15 | type: 'junit', 16 | artifact_path: 'foo' 17 | ] 18 | end 19 | 20 | it 'does not call annotate' do 21 | run 22 | expect(agent_annotate_commands).to be_empty 23 | end 24 | end 25 | 26 | context 'with failures' do 27 | let(:inputs) do 28 | [ 29 | label: 'rspec', 30 | type: 'junit', 31 | artifact_path: 'rspec*' 32 | ] 33 | end 34 | 35 | it 'calls annotate with correct args' do 36 | run 37 | expect(agent_annotate_commands.first).to include('annotate', '--context', 'test-summary', '--style', 'error') 38 | end 39 | 40 | context 'with custom style' do 41 | let(:params) { { inputs: inputs, style: 'warning' } } 42 | 43 | it 'calls annotate with correct args' do 44 | run 45 | expect(agent_annotate_commands.first).to include('annotate', '--context', 'test-summary', '--style', 'warning') 46 | end 47 | end 48 | 49 | context 'with custom context' do 50 | let(:params) { { inputs: inputs, context: 'ponies' } } 51 | 52 | it 'calls annotate with correct args' do 53 | run 54 | expect(agent_annotate_commands.first).to include('annotate', '--context', 'ponies', '--style', 'error') 55 | end 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /spec/test_summary_buildkite_plugin/tap_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | RSpec.describe TestSummaryBuildkitePlugin::Tap do 6 | subject(:suite) { described_class::Parser.new(text).parse } 7 | 8 | context 'sane version 12' do 9 | let(:text) { "not ok 17 - failure\nok 18 - yay" } 10 | 11 | it 'is version 12' do 12 | expect(suite).to have_attributes(version: 12) 13 | end 14 | 15 | it 'contains the tests' do 16 | expect(suite.tests).to(match_array([ 17 | have_attributes(passed: false, description: 'failure'), 18 | have_attributes(passed: true, description: 'yay') 19 | ])) 20 | end 21 | end 22 | 23 | context 'sane version 13' do 24 | let(:text) { "TAP version 13\nnot ok 10 failure\nok 18 - yay" } 25 | 26 | it 'is version 13' do 27 | expect(suite).to have_attributes(version: 13) 28 | end 29 | 30 | it 'contains the tests' do 31 | expect(suite.tests).to(match_array([ 32 | have_attributes(passed: false, description: 'failure'), 33 | have_attributes(passed: true, description: 'yay') 34 | ])) 35 | end 36 | end 37 | 38 | context 'with todo tests' do 39 | let(:text) { 'not ok be awesome # TODO get around to this' } 40 | 41 | it 'is handled correctly' do 42 | expect(suite.tests.first).to have_attributes( 43 | passed: false, 44 | description: 'be awesome', 45 | directive: 'TODO get around to this', 46 | todo: be_truthy, 47 | skipped: be_falsey 48 | ) 49 | end 50 | end 51 | 52 | context 'with skipped tests' do 53 | let(:text) { 'not ok be awesome # SKIP get around to this' } 54 | 55 | it 'is handled correctly' do 56 | expect(suite.tests.first).to have_attributes( 57 | passed: false, 58 | description: 'be awesome', 59 | directive: 'SKIP get around to this', 60 | todo: be_falsey, 61 | skipped: be_truthy 62 | ) 63 | end 64 | end 65 | 66 | context 'with lowercase skipped tests' do 67 | let(:text) { 'not ok be awesome # skipped get around to this' } 68 | 69 | it 'is handled correctly' do 70 | expect(suite.tests.first).to have_attributes( 71 | passed: false, 72 | description: 'be awesome', 73 | directive: 'skipped get around to this', 74 | todo: be_falsey, 75 | skipped: be_truthy 76 | ) 77 | end 78 | end 79 | 80 | context 'with indented directives' do 81 | let(:text) { "not ok be awesome\n# two\n# four" } 82 | 83 | it 'unindents' do 84 | expect(suite.tests.first).to have_attributes(diagnostic: "two\n four") 85 | end 86 | end 87 | 88 | context 'with yaml' do 89 | let(:text) { "not ok be awesome\n ---\n one:\n two\n ..." } 90 | 91 | it 'unindents' do 92 | expect(suite.tests.first).to have_attributes(yaml: "one:\n two") 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /spec/test_summary_buildkite_plugin/truncater_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | RSpec.describe TestSummaryBuildkitePlugin::Truncater do 6 | let(:max_size) { 5000 } 7 | let(:input1) { double(TestSummaryBuildkitePlugin::Input::Base, label: 'animals') } 8 | let(:input2) { double(TestSummaryBuildkitePlugin::Input::Base, label: 'cars') } 9 | let(:inputs) { [input1, input2] } 10 | let(:failures1) do 11 | %w[dog cat pony horse unicorn].map { |x| TestSummaryBuildkitePlugin::Failure::Unstructured.new(x) } 12 | end 13 | let(:failures2) do 14 | %w[toyota honda holden ford mazda volkswagen].map { |x| TestSummaryBuildkitePlugin::Failure::Unstructured.new(x) } 15 | end 16 | let(:formatter_opts) { nil } 17 | let(:fail_on_error) { false } 18 | let(:options) { { max_size: max_size, inputs: inputs, formatter_opts: formatter_opts, fail_on_error: fail_on_error } } 19 | let(:truncater) { described_class.new(options) } 20 | 21 | subject(:truncated) { truncater.markdown } 22 | 23 | before do 24 | allow(input1).to receive(:failures).and_return(failures1) 25 | allow(input2).to receive(:failures).and_return(failures2) 26 | end 27 | 28 | context 'when no failures' do 29 | let(:failures1) { [] } 30 | let(:failures2) { [] } 31 | 32 | it 'returns nothing' do 33 | is_expected.to be_empty 34 | end 35 | end 36 | 37 | context 'when below max size' do 38 | let(:max_size) { 50_000 } 39 | 40 | it 'returns all failures' do 41 | is_expected.to include('dog', 'cat', 'pony', 'horse', 'unicorn') 42 | is_expected.to include('toyota', 'honda', 'holden', 'ford', 'mazda', 'volkswagen') 43 | end 44 | end 45 | 46 | context 'when above max size' do 47 | let(:max_size) { 400 } 48 | 49 | it 'returns something below max size' do 50 | expect(truncated.bytesize).to be <= max_size 51 | end 52 | 53 | it 'optimally truncates' do 54 | is_expected.to include('Including first 4') 55 | end 56 | end 57 | 58 | context 'when optimization fails' do 59 | before do 60 | allow(truncater).to receive(:markdown_with_truncation).and_return('a' * (max_size + 1)) 61 | end 62 | 63 | it 'shows a helpful error' do 64 | is_expected.to include('ANNOTATION ERROR') 65 | end 66 | end 67 | 68 | context 'formatter raises exceptions' do 69 | let(:formatter) { spy } 70 | 71 | before do 72 | allow(truncater).to receive(:formatter).and_return(formatter) 73 | allow(formatter).to receive(:markdown).with(input1).and_raise('life sucks') 74 | allow(formatter).to receive(:markdown).with(input2).and_return('awesome markdown') 75 | end 76 | 77 | context 'without fail_on_error' do 78 | it 'continues' do 79 | expect(truncated).to include('awesome markdown') 80 | end 81 | 82 | it 'logs the error' do 83 | expect { truncated }.to output(/life sucks/).to_stdout 84 | end 85 | end 86 | 87 | context 'with fail_on_error' do 88 | let(:fail_on_error) { true } 89 | 90 | it 'raises error' do 91 | expect { truncated }.to raise_error('life sucks') 92 | end 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /spec/test_summary_buildkite_plugin_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe TestSummaryBuildkitePlugin do 4 | it 'has a version number' do 5 | expect(TestSummaryBuildkitePlugin::VERSION).not_to be nil 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /templates/details/failures.html.haml: -------------------------------------------------------------------------------- 1 | %ul 2 | - failures.each do |failure| 3 | %li 4 | - if failure.message || failure.details || failure.job_id 5 | %details 6 | %summary 7 | %code 8 | = failure.summary 9 | %dl 10 | - if failure.message 11 | %dt Message 12 | %dd 13 | %pre 14 | %code 15 | = failure.message 16 | - if failure.details 17 | %dt Details 18 | %dd 19 | %pre 20 | %code 21 | = failure.details 22 | - if failure.job_id 23 | %dt Job 24 | %dd 25 | %a{href: "##{failure.job_id}"} 26 | = failure.job_id 27 | - else 28 | %code 29 | = failure.summary 30 | -------------------------------------------------------------------------------- /templates/summary/failures.html.haml: -------------------------------------------------------------------------------- 1 | %pre 2 | %code 3 | = failures.map(&:summary).join("\n") 4 | -------------------------------------------------------------------------------- /templates/summary/footer.html.haml: -------------------------------------------------------------------------------- 1 | -# Note: we need to use markdown links here rather than 's otherwise the markdown renderer adds incorrect spacing 2 | - unless job_ids.empty? 3 | See 4 | - job_ids[0...-2].each do |job_id| 5 | = "[job #{job_id}](##{job_id})," 6 | 7 | - if job_ids.count > 1 8 | = "[job #{job_ids[-2]}](##{job_ids[-2]})" 9 | and 10 | 11 | = "[job #{job_ids.last}](##{job_ids.last})" 12 | -------------------------------------------------------------------------------- /templates/test_layout.html.haml: -------------------------------------------------------------------------------- 1 | !!! 2 | %html.tiny-kitemoji{lang: "en"} 3 | %head 4 | %meta{content: "text/html; charset=UTF-8", "http-equiv" => "Content-Type"}/ 5 | %meta{charset: "utf-8"}/ 6 | %meta{content: "width=device-width, initial-scale=1.0", name: "viewport"}/ 7 | %title Test markdown 8 | %link{href: "https://buildkiteassets.com/assets/vendor-8704755f7aaa0c5d358ba4643640cd76adff6bdee8ed8e4598766fa845535028.css", media: "screen", rel: "stylesheet"}/ 9 | %link{href: "https://buildkiteassets.com/assets/application-a8fe66f67c4e64d2d8b8d8b57468454153729fa7069eacc30bb23edcba574db8.css", media: "screen", rel: "stylesheet"}/ 10 | %link{href: "https://buildkiteassets.com/frontend/app-dd015274521a66a3f467.css", rel: "stylesheet"}/ 11 | %meta{content: "origin-when-cross-origin", name: "referrer"}/ 12 | %body.new-nav 13 | .twbs-container 14 | #build 15 | %div 16 | %div 17 | .rounded.flex.items-stretch.border-red.border.mb4 18 | .bg-red.flex-none.flex.pt3{style: "width: 30px;"} 19 | .center.flex-auto 20 | %i.fa.fa-times.white 21 | .flex-auto 22 | .m3.annotation-body 23 | != content 24 | -------------------------------------------------------------------------------- /templates/truncater_exception.html.haml: -------------------------------------------------------------------------------- 1 | **ANNOTATION ERROR** The annotation exceeded the maximum size and truncation optimization failed. 2 | This should never happen so if it has, please report it do 3 | %a{href: 'https://github.com/bugcrowd/test-summary-buildkite-plugin/issues'} 4 | https://github.com/bugcrowd/test-summary-buildkite-plugin/issues 5 | with the diagnostic information included in the log. 6 | --------------------------------------------------------------------------------