├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── cla.yaml │ ├── lint.yaml │ ├── tag.yaml │ └── test.yaml ├── .gitignore ├── LICENSE ├── README.md ├── action.yml ├── example.png └── fixtures ├── README.md ├── test-baseline-main.xml ├── test-branch.xml ├── test-comprehensive.xml ├── test-coverage-improved.xml ├── test-coverage-rate-reduction.xml ├── test-exception-disabled.xml ├── test-missing-lines-2.xml ├── test-missing-lines.xml ├── test-new-uncovered-statements.xml ├── test-no-branch-2.xml ├── test-no-branch.xml ├── test-no-uncovered-statements.xml ├── test-pycobertura-exception.xml ├── test-python.xml ├── test-togglable-report.xml ├── test-uncovered-statements-decreased.xml └── test-uncovered-statements-increase.xml /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @insightsengineering/idr 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /.github/workflows/cla.yaml: -------------------------------------------------------------------------------- 1 | name: CLA 🔏 2 | 3 | on: 4 | issue_comment: 5 | types: 6 | - created 7 | # For PRs that originate from forks 8 | pull_request_target: 9 | types: 10 | - opened 11 | - closed 12 | - synchronize 13 | 14 | jobs: 15 | CLA: 16 | name: CLA 📝 17 | uses: insightsengineering/.github/.github/workflows/cla.yaml@main 18 | secrets: inherit 19 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: SuperLinter 🦸‍♀️ 3 | 4 | on: 5 | pull_request: 6 | branches: 7 | - main 8 | 9 | jobs: 10 | lint: 11 | name: Lint Code Base 🧶 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout Code 🛎 15 | uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | 19 | - name: Lint Code Base 👟 20 | uses: github/super-linter/slim@v5 21 | env: 22 | VALIDATE_ALL_CODEBASE: false 23 | DEFAULT_BRANCH: main 24 | VALIDATE_YAML: true 25 | VALIDATE_GITHUB_ACTIONS: true 26 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 27 | -------------------------------------------------------------------------------- /.github/workflows/tag.yaml: -------------------------------------------------------------------------------- 1 | name: Keep the versions up-to-date ☕️ 2 | 3 | on: 4 | release: 5 | types: [published, edited] 6 | 7 | jobs: 8 | actions-tagger: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Create additional tags 🎟 12 | uses: Actions-R-Us/actions-tagger@latest 13 | with: 14 | publish_latest_tag: true 15 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Test 🧪 3 | 4 | on: 5 | pull_request: 6 | branches: 7 | - main 8 | push: 9 | branches: 10 | - main 11 | 12 | jobs: 13 | test: 14 | name: Test action 🎬 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | reports: 20 | - path: test-branch.xml 21 | threshold: 80 22 | fail: true 23 | publish: false 24 | diff: false 25 | diff-branch: main 26 | diff-storage: _xml_coverage_reports 27 | coverage-summary-title: "Code Coverage Summary (test-branch.xml)" 28 | exclude-detailed-coverage: false 29 | uncovered-statements-increase-failure: false 30 | new-uncovered-statements-failure: false 31 | coverage-rate-reduction-failure: false 32 | pycobertura-exception-failure: true 33 | togglable-report: false 34 | - path: test-missing-lines.xml 35 | threshold: 50 36 | fail: false 37 | publish: true 38 | diff: false 39 | diff-branch: main 40 | diff-storage: _xml_coverage_reports_5 41 | coverage-summary-title: "Code Coverage Summary (test-missing-lines.xml)" 42 | exclude-detailed-coverage: false 43 | uncovered-statements-increase-failure: false 44 | new-uncovered-statements-failure: false 45 | coverage-rate-reduction-failure: false 46 | pycobertura-exception-failure: true 47 | togglable-report: false 48 | - path: test-no-branch.xml 49 | threshold: 90 50 | fail: true 51 | publish: true 52 | diff: true 53 | diff-branch: main 54 | diff-storage: _xml_coverage_reports_1 55 | coverage-summary-title: "Code Coverage Summary (test-no-branch.xml)" 56 | exclude-detailed-coverage: false 57 | uncovered-statements-increase-failure: false 58 | new-uncovered-statements-failure: false 59 | coverage-rate-reduction-failure: false 60 | pycobertura-exception-failure: true 61 | togglable-report: false 62 | - path: test-python.xml 63 | threshold: 90 64 | fail: false 65 | publish: false 66 | diff: true 67 | diff-branch: main 68 | diff-storage: _xml_coverage_reports_2 69 | coverage-summary-title: "Code Coverage Summary (test-python.xml)" 70 | exclude-detailed-coverage: false 71 | uncovered-statements-increase-failure: false 72 | new-uncovered-statements-failure: false 73 | coverage-rate-reduction-failure: false 74 | pycobertura-exception-failure: true 75 | togglable-report: false 76 | - path: test-no-branch-2.xml 77 | threshold: 90 78 | fail: true 79 | publish: true 80 | diff: true 81 | diff-branch: main 82 | diff-storage: _xml_coverage_reports_3 83 | coverage-summary-title: "Code Coverage Summary (test-no-branch.xml) without detailed coverage" 84 | exclude-detailed-coverage: true 85 | uncovered-statements-increase-failure: false 86 | new-uncovered-statements-failure: false 87 | coverage-rate-reduction-failure: false 88 | pycobertura-exception-failure: true 89 | togglable-report: false 90 | - path: test-missing-lines-2.xml 91 | threshold: 50 92 | fail: false 93 | publish: true 94 | diff: false 95 | diff-branch: main 96 | diff-storage: _xml_coverage_reports_4 97 | coverage-summary-title: "Code Coverage Summary (test-missing-lines.xml) without detailed coverage" 98 | exclude-detailed-coverage: true 99 | uncovered-statements-increase-failure: false 100 | new-uncovered-statements-failure: false 101 | coverage-rate-reduction-failure: false 102 | pycobertura-exception-failure: true 103 | togglable-report: false 104 | # New test cases for uncovered-statements-increase-failure 105 | - path: test-uncovered-statements-increase.xml 106 | threshold: 50 107 | fail: false 108 | publish: false 109 | diff: true 110 | diff-branch: main 111 | diff-storage: _xml_coverage_reports_uncovered_increase 112 | coverage-summary-title: "Code Coverage Summary (uncovered statements increase)" 113 | exclude-detailed-coverage: false 114 | uncovered-statements-increase-failure: true 115 | new-uncovered-statements-failure: false 116 | coverage-rate-reduction-failure: false 117 | pycobertura-exception-failure: true 118 | togglable-report: false 119 | # New test cases for new-uncovered-statements-failure 120 | - path: test-new-uncovered-statements.xml 121 | threshold: 50 122 | fail: false 123 | publish: false 124 | diff: true 125 | diff-branch: main 126 | diff-storage: _xml_coverage_reports_new_uncovered 127 | coverage-summary-title: "Code Coverage Summary (new uncovered statements)" 128 | exclude-detailed-coverage: false 129 | uncovered-statements-increase-failure: false 130 | new-uncovered-statements-failure: true 131 | coverage-rate-reduction-failure: false 132 | pycobertura-exception-failure: true 133 | togglable-report: false 134 | # New test cases for coverage-rate-reduction-failure 135 | - path: test-coverage-rate-reduction.xml 136 | threshold: 50 137 | fail: false 138 | publish: false 139 | diff: true 140 | diff-branch: main 141 | diff-storage: _xml_coverage_reports_rate_reduction 142 | coverage-summary-title: "Code Coverage Summary (coverage rate reduction)" 143 | exclude-detailed-coverage: false 144 | uncovered-statements-increase-failure: false 145 | new-uncovered-statements-failure: false 146 | coverage-rate-reduction-failure: true 147 | pycobertura-exception-failure: true 148 | togglable-report: false 149 | # New test cases for togglable-report 150 | - path: test-togglable-report.xml 151 | threshold: 80 152 | fail: false 153 | publish: true 154 | diff: false 155 | diff-branch: main 156 | diff-storage: _xml_coverage_reports_togglable 157 | coverage-summary-title: "Code Coverage Summary (togglable report)" 158 | exclude-detailed-coverage: false 159 | uncovered-statements-increase-failure: false 160 | new-uncovered-statements-failure: false 161 | coverage-rate-reduction-failure: false 162 | pycobertura-exception-failure: true 163 | togglable-report: true 164 | # Test case with multiple failure modes enabled 165 | - path: test-branch.xml 166 | threshold: 80 167 | fail: true 168 | publish: false 169 | diff: true 170 | diff-branch: main 171 | diff-storage: _xml_coverage_reports_multiple_failures 172 | coverage-summary-title: "Code Coverage Summary (multiple failure modes)" 173 | exclude-detailed-coverage: false 174 | uncovered-statements-increase-failure: true 175 | new-uncovered-statements-failure: true 176 | coverage-rate-reduction-failure: true 177 | pycobertura-exception-failure: true 178 | togglable-report: false 179 | # Test case with togglable report and excluded detailed coverage 180 | - path: test-togglable-report.xml 181 | threshold: 80 182 | fail: false 183 | publish: true 184 | diff: false 185 | diff-branch: main 186 | diff-storage: _xml_coverage_reports_togglable_excluded 187 | coverage-summary-title: "Code Coverage Summary (togglable report, excluded details)" 188 | exclude-detailed-coverage: true 189 | uncovered-statements-increase-failure: false 190 | new-uncovered-statements-failure: false 191 | coverage-rate-reduction-failure: false 192 | pycobertura-exception-failure: true 193 | togglable-report: true 194 | # Test case for pycobertura exception (diff scenario) 195 | - path: test-pycobertura-exception.xml 196 | threshold: 50 197 | fail: false 198 | publish: false 199 | diff: true 200 | diff-branch: main 201 | diff-storage: _xml_coverage_reports_exception 202 | coverage-summary-title: "Code Coverage Summary (pycobertura exception flag enabled)" 203 | exclude-detailed-coverage: false 204 | uncovered-statements-increase-failure: false 205 | new-uncovered-statements-failure: false 206 | coverage-rate-reduction-failure: false 207 | pycobertura-exception-failure: true 208 | togglable-report: false 209 | # Test case for improved coverage (should not fail coverage-rate-reduction-failure) 210 | - path: test-coverage-improved.xml 211 | threshold: 80 212 | fail: false 213 | publish: false 214 | diff: true 215 | diff-branch: main 216 | diff-storage: _xml_coverage_reports_improved 217 | coverage-summary-title: "Code Coverage Summary (improved coverage)" 218 | exclude-detailed-coverage: false 219 | uncovered-statements-increase-failure: false 220 | new-uncovered-statements-failure: false 221 | coverage-rate-reduction-failure: true 222 | pycobertura-exception-failure: true 223 | togglable-report: false 224 | # Test case for no uncovered statements (should not fail new-uncovered-statements-failure) 225 | - path: test-no-uncovered-statements.xml 226 | threshold: 100 227 | fail: false 228 | publish: false 229 | diff: true 230 | diff-branch: main 231 | diff-storage: _xml_coverage_reports_no_uncovered 232 | coverage-summary-title: "Code Coverage Summary (no uncovered statements)" 233 | exclude-detailed-coverage: false 234 | uncovered-statements-increase-failure: false 235 | new-uncovered-statements-failure: true 236 | coverage-rate-reduction-failure: false 237 | pycobertura-exception-failure: true 238 | togglable-report: false 239 | # Test case for decreased uncovered statements (should not fail uncovered-statements-increase-failure) 240 | - path: test-uncovered-statements-decreased.xml 241 | threshold: 90 242 | fail: false 243 | publish: false 244 | diff: true 245 | diff-branch: main 246 | diff-storage: _xml_coverage_reports_decreased_uncovered 247 | coverage-summary-title: "Code Coverage Summary (decreased uncovered statements)" 248 | exclude-detailed-coverage: false 249 | uncovered-statements-increase-failure: true 250 | new-uncovered-statements-failure: false 251 | coverage-rate-reduction-failure: false 252 | pycobertura-exception-failure: true 253 | togglable-report: false 254 | # Test case for comprehensive scenario with multiple files 255 | - path: test-comprehensive.xml 256 | threshold: 70 257 | fail: false 258 | publish: true 259 | diff: true 260 | diff-branch: main 261 | diff-storage: _xml_coverage_reports_comprehensive 262 | coverage-summary-title: "Code Coverage Summary (comprehensive test)" 263 | exclude-detailed-coverage: false 264 | uncovered-statements-increase-failure: false 265 | new-uncovered-statements-failure: false 266 | coverage-rate-reduction-failure: false 267 | pycobertura-exception-failure: true 268 | togglable-report: true 269 | # Test case for exception disabled 270 | - path: test-exception-disabled.xml 271 | threshold: 80 272 | fail: false 273 | publish: false 274 | diff: false 275 | diff-branch: main 276 | diff-storage: _xml_coverage_reports_exception_disabled 277 | coverage-summary-title: "Code Coverage Summary (exception disabled)" 278 | exclude-detailed-coverage: false 279 | uncovered-statements-increase-failure: false 280 | new-uncovered-statements-failure: false 281 | coverage-rate-reduction-failure: false 282 | pycobertura-exception-failure: false 283 | togglable-report: false 284 | steps: 285 | - name: Checkout Code 🛎 286 | uses: actions/checkout@v4 287 | 288 | - name: Run test on ${{ matrix.reports.path }} 🏃‍♀️ 289 | uses: ./ 290 | with: 291 | path: ./fixtures/${{ matrix.reports.path }} 292 | threshold: ${{ matrix.reports.threshold }} 293 | fail: ${{ matrix.reports.fail }} 294 | publish: ${{ matrix.reports.publish }} 295 | diff: ${{ matrix.reports.diff }} 296 | diff-branch: ${{ matrix.reports.diff-branch }} 297 | diff-storage: ${{ matrix.reports.diff-storage }} 298 | coverage-summary-title: ${{ matrix.reports.coverage-summary-title }} 299 | exclude-detailed-coverage: ${{ matrix.reports.exclude-detailed-coverage }} 300 | uncovered-statements-increase-failure: ${{ matrix.reports.uncovered-statements-increase-failure }} 301 | new-uncovered-statements-failure: ${{ matrix.reports.new-uncovered-statements-failure }} 302 | coverage-rate-reduction-failure: ${{ matrix.reports.coverage-rate-reduction-failure }} 303 | pycobertura-exception-failure: ${{ matrix.reports.pycobertura-exception-failure }} 304 | togglable-report: ${{ matrix.reports.togglable-report }} 305 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Roche/Genentech - Insights Engineering 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Code Coverage Report Action 2 | 3 | [![SuperLinter](https://github.com/insightsengineering/coverage-action/actions/workflows/lint.yaml/badge.svg)](https://github.com/insightsengineering/coverage-action/actions/workflows/lint.yaml) 4 | [![Test](https://github.com/insightsengineering/coverage-action/actions/workflows/test.yaml/badge.svg)](https://github.com/insightsengineering/coverage-action/actions/workflows/test.yaml) 5 | 6 | Action that converts a Cobertura XML report into a markdown report. 7 | 8 | ## Action Type 9 | 10 | Composite 11 | 12 | ## Author 13 | 14 | Inisghts Engineering 15 | 16 | ## Inputs 17 | 18 | * `token`: 19 | 20 | _Description_: Github token to use to publish the check. 21 | 22 | _Required_: `false` 23 | 24 | _Default_: `${{ github.token }}` 25 | 26 | * `path`: 27 | 28 | _Description_: Path to the Cobertura coverage XML report. 29 | 30 | _Required_: `false` 31 | 32 | _Default_: `coverage.xml` 33 | 34 | * `threshold`: 35 | 36 | _Description_: The minimum allowed coverage percentage, as a real number. 37 | 38 | _Required_: `false` 39 | 40 | _Default_: `0` 41 | 42 | * `fail`: 43 | 44 | _Description_: Fail the action when the minimum coverage was not met. 45 | 46 | _Required_: `false` 47 | 48 | _Default_: `True` 49 | 50 | * `publish`: 51 | 52 | _Description_: Publish the coverage report as an issue comment. 53 | 54 | _Required_: `false` 55 | 56 | _Default_: `False` 57 | 58 | * `diff`: 59 | 60 | _Description_: Create a diff of the coverage report. 61 | 62 | _Required_: `false` 63 | 64 | _Default_: `False` 65 | 66 | * `diff-branch`: 67 | 68 | _Description_: Branch to diff against. 69 | 70 | _Required_: `false` 71 | 72 | _Default_: `main` 73 | 74 | * `storage-subdirectory`: 75 | 76 | _Description_: Subdirectory in the diff-storage branch where the XML reports will be stored. 77 | 78 | _Required_: `false` 79 | 80 | _Default_: `.` 81 | 82 | * `diff-storage`: 83 | 84 | _Description_: Branch where coverage reports are stored for diff purposes. 85 | 86 | _Required_: `false` 87 | 88 | _Default_: `_xml_coverage_reports` 89 | 90 | * `coverage-summary-title`: 91 | 92 | _Description_: Title for the code coverage summary in the Pull Request comment. 93 | 94 | _Required_: `false` 95 | 96 | _Default_: `Code Coverage Summary` 97 | 98 | * `uncovered-statements-increase-failure`: 99 | 100 | _Description_: Fail the action if any changed file has an increase in uncovered lines compared to the `diff-branch`. This corresponds to pycobertura exit code 2, which indicates that at least one changed file has more uncovered lines than before (Miss > 0). Note that this is different from coverage rate reduction - it specifically checks for increases in the absolute number of uncovered lines. 101 | 102 | _Required_: `false` 103 | 104 | _Default_: `False` 105 | 106 | * `new-uncovered-statements-failure`: 107 | 108 | _Description_: Fail the action if new uncovered statements are introduced AND overall coverage improved (total uncovered lines decreased) compared to the `diff-branch`. This corresponds to pycobertura exit code 3, which only occurs when total uncovered lines decreased (Miss <= 0) but there are still new uncovered statements (Missing != []). To fail on ALL new uncovered statements regardless of overall coverage improvement, use this flag together with `uncovered-statements-increase-failure: true`. 109 | 110 | _Required_: `false` 111 | 112 | _Default_: `False` 113 | 114 | * `coverage-rate-reduction-failure`: 115 | 116 | _Description_: Fail the action if the overall coverage percentage (rate) decreases compared to the `diff-branch`. This is different from `uncovered-statements-increase-failure` which checks for absolute increases in uncovered lines. This flag specifically looks at the coverage percentage and fails if it goes down, regardless of whether uncovered lines increased or decreased. This is a more forgiving approach that focuses on the relative coverage rate rather than absolute uncovered line counts. 117 | 118 | _Required_: `false` 119 | 120 | _Default_: `False` 121 | 122 | * `pycobertura-exception-failure`: 123 | 124 | _Description_: Fail the action in case of a `Pycobertura` exception. 125 | 126 | _Required_: `false` 127 | 128 | _Default_: `True` 129 | 130 | * `togglable-report`: 131 | 132 | _Description_: Make the code coverage report togglable. 133 | 134 | _Required_: `false` 135 | 136 | _Default_: `False` 137 | 138 | * `exclude-detailed-coverage`: 139 | 140 | _Description_: Whether a detailed coverage report should be excluded from the PR comment. 141 | The detailed coverage report contains the following information per file: 142 | number of code statements, number of statements not covered by any test, 143 | coverage percentage, and line numbers not covered by any test. 144 | 145 | _Required_: `false` 146 | 147 | _Default_: `False` 148 | 149 | ### Outputs 150 | 151 | * `summary`: 152 | 153 | _Description_: A summary of coverage report 154 | 155 | ## How it works 156 | 157 | This tool makes use of the [PyCobertura](https://github.com/aconrad/pycobertura) CLI tool to produce the summary outputs. The action also supports `diff`s against a given branch and makes use of a remote branch to store reports, which can be specified via this action. 158 | 159 | ## Failure Modes 160 | 161 | The action provides three different failure modes for detecting coverage regressions, each with different characteristics: 162 | 163 | ### 1. `uncovered-statements-increase-failure` (Strict) 164 | 165 | * **When it fails**: When any changed file has an increase in the absolute number of uncovered lines 166 | * **Pycobertura exit code**: 2 167 | * **Use case**: When you want to ensure that no file gets worse coverage, regardless of overall improvements 168 | * **Example**: If you add 5 uncovered lines to file A but remove 10 uncovered lines from file B, this will still fail because file A got worse 169 | 170 | ### 2. `new-uncovered-statements-failure` (Moderate) 171 | 172 | * **When it fails**: When new uncovered statements are introduced AND overall coverage improved 173 | * **Pycobertura exit code**: 3 174 | * **Use case**: When you want to allow overall improvements but still catch new uncovered code 175 | * **Example**: If you add 5 uncovered lines to file A but remove 10 uncovered lines from file B, this will fail because there are new uncovered statements, even though overall coverage improved 176 | 177 | ### 3. `coverage-rate-reduction-failure` (Forgiving) 178 | 179 | * **When it fails**: When the overall coverage percentage decreases 180 | * **Pycobertura exit code**: N/A (custom implementation) 181 | * **Use case**: When you want to focus on the overall coverage rate rather than absolute line counts 182 | * **Example**: If you add 5 uncovered lines to file A but remove 10 uncovered lines from file B, this will pass if the overall coverage percentage improved 183 | 184 | ### Combining Failure Modes 185 | 186 | You can combine these failure modes for different levels of strictness: 187 | 188 | ```yaml 189 | # Most strict: Fail on any uncovered line increase 190 | uncovered-statements-increase-failure: true 191 | new-uncovered-statements-failure: true 192 | coverage-rate-reduction-failure: true 193 | 194 | # Moderate: Allow overall improvements but catch new uncovered code 195 | uncovered-statements-increase-failure: false 196 | new-uncovered-statements-failure: true 197 | coverage-rate-reduction-failure: true 198 | 199 | # Most forgiving: Only fail if overall coverage percentage decreases 200 | uncovered-statements-increase-failure: false 201 | new-uncovered-statements-failure: false 202 | coverage-rate-reduction-failure: true 203 | ``` 204 | 205 | ## Usage 206 | 207 | Example usage: 208 | 209 | ```yaml 210 | --- 211 | name: Code Coverage 212 | 213 | on: 214 | # NOTE: Both, the 'pull_request' and the 'push' 215 | # events are REQUIRED to take full advantage 216 | # of the features of this action. 217 | pull_request: 218 | branches: 219 | - main 220 | push: 221 | branches: 222 | - main 223 | 224 | jobs: 225 | coverage: 226 | name: Calculate code coverage 227 | runs-on: ubuntu-latest 228 | 229 | steps: 230 | - name: Checkout Code 231 | uses: actions/checkout@v3 232 | 233 | - name: Your logic to generate the Cobertura XML goes here 234 | run: echo "Your logic to generate the Cobertura XML goes here" 235 | 236 | - name: Produce the coverage report 237 | uses: insightsengineering/coverage-action@v3 238 | with: 239 | # Path to the Cobertura XML report. 240 | path: ./cobertura.xml 241 | # Minimum total coverage, if you want to the 242 | # workflow to enforce it as a standard. 243 | # This has no effect if the `fail` arg is set to `false`. 244 | threshold: 80.123 245 | # Fail the workflow if the minimum code coverage 246 | # reuqirements are not satisfied. 247 | fail: true 248 | # Publish the rendered output as a PR comment 249 | publish: true 250 | # Create a coverage diff report. 251 | diff: true 252 | # Branch to diff against. 253 | # Compare the current coverage to the coverage 254 | # determined on this branch. 255 | diff-branch: main 256 | # This is where the coverage reports for the 257 | # `diff-branch` are stored. 258 | # Branch is created if it doesn't already exist'. 259 | diff-storage: _xml_coverage_reports 260 | # A custom title that can be added to the code 261 | # coverage summary in the PR comment. 262 | coverage-summary-title: "Code Coverage Summary" 263 | # Failure modes for coverage regression detection: 264 | # Fail if any changed file has more uncovered lines (pycobertura exit code 2) 265 | uncovered-statements-increase-failure: false 266 | # Fail if new uncovered statements are introduced despite overall improvement (pycobertura exit code 3) 267 | new-uncovered-statements-failure: false 268 | # Fail if the overall coverage percentage decreases (more forgiving approach) 269 | coverage-rate-reduction-failure: true 270 | ``` 271 | 272 | An example of the output of the action can be seen below: 273 | 274 | ![Action output](example.png) 275 | -------------------------------------------------------------------------------- /action.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # yamllint disable rule:line-length 3 | name: Code Coverage Report Action 4 | author: Inisghts Engineering 5 | description: Action that converts a Cobertura XML report into a markdown report. 6 | 7 | inputs: 8 | token: 9 | description: Github token to use to publish the check. 10 | required: false 11 | default: ${{ github.token }} 12 | path: 13 | description: Path to the Cobertura coverage XML report. 14 | required: false 15 | default: coverage.xml 16 | threshold: 17 | description: The minimum allowed coverage percentage, as a real number. 18 | required: false 19 | default: 0 20 | fail: 21 | description: Fail the action when the minimum coverage was not met. 22 | required: false 23 | default: true 24 | publish: 25 | description: Publish the coverage report as an issue comment. 26 | required: false 27 | default: false 28 | diff: 29 | description: Create a diff of the coverage report. 30 | required: false 31 | default: false 32 | diff-branch: 33 | description: Branch to diff against. 34 | required: false 35 | default: main 36 | storage-subdirectory: 37 | description: Subdirectory in the diff-storage branch where the XML reports will be stored. 38 | required: false 39 | default: "." 40 | diff-storage: 41 | description: Branch where coverage reports are stored for diff purposes. 42 | required: false 43 | default: _xml_coverage_reports 44 | coverage-summary-title: 45 | description: Title for the code coverage summary in the Pull Request comment. 46 | required: false 47 | default: "Code Coverage Summary" 48 | uncovered-statements-increase-failure: 49 | description: | 50 | Fail the action if any changed file has an increase in uncovered lines compared to the `diff-branch`. 51 | This corresponds to pycobertura exit code 2, which indicates that at least one changed file has 52 | more uncovered lines than before (Miss > 0). Note that this is different from coverage rate 53 | reduction - it specifically checks for increases in the absolute number of uncovered lines. 54 | required: false 55 | default: false 56 | new-uncovered-statements-failure: 57 | description: | 58 | Fail the action if new uncovered statements are introduced AND overall coverage improved 59 | (total uncovered lines decreased) compared to the `diff-branch`. 60 | This corresponds to pycobertura exit code 3, which only occurs when total uncovered lines 61 | decreased (Miss <= 0) but there are still new uncovered statements (Missing != []). 62 | To fail on ALL new uncovered statements regardless of overall coverage improvement, 63 | use this flag together with `coverage-reduction-failure: true`. 64 | required: false 65 | default: false 66 | coverage-rate-reduction-failure: 67 | description: | 68 | Fail the action if the overall coverage percentage (rate) decreases compared to the `diff-branch`. 69 | This is different from `uncovered-statements-increase-failure` which checks for absolute 70 | increases in uncovered lines. This flag specifically looks at the coverage percentage 71 | and fails if it goes down, regardless of whether uncovered lines increased or decreased. 72 | This is a more forgiving approach that focuses on the relative coverage rate rather than 73 | absolute uncovered line counts. 74 | required: false 75 | default: false 76 | pycobertura-exception-failure: 77 | description: Fail the action in case of a `Pycobertura` exception. 78 | required: false 79 | default: true 80 | togglable-report: 81 | description: Make the code coverage report togglable. 82 | required: false 83 | default: false 84 | exclude-detailed-coverage: 85 | description: | 86 | Whether a detailed coverage report should be excluded from the PR comment. 87 | The detailed coverage report contains the following information per file: 88 | number of code statements, number of statements not covered by any test, 89 | coverage percentage, and line numbers not covered by any test. 90 | required: false 91 | default: false 92 | 93 | outputs: 94 | summary: 95 | description: Summary of coverage report 96 | value: ${{ steps.create-output.outputs.summary }} 97 | 98 | branding: # https://feathericons.com/ 99 | icon: "umbrella" 100 | color: "red" 101 | 102 | runs: 103 | using: composite 104 | steps: 105 | - name: Setup Python 106 | uses: actions/setup-python@v5 107 | with: 108 | python-version: '3.11' 109 | 110 | - name: Install pycobertura 111 | uses: insightsengineering/pip-action@v2 112 | with: 113 | packages: pycobertura==3.0.0 114 | 115 | - name: Get branch names 116 | id: branch-names 117 | uses: tj-actions/branch-names@v7 118 | 119 | - name: Generate text report 120 | run: | 121 | mkdir -p coverage-action 122 | cp ${{ inputs.path }} coverage-action/ 123 | pycobertura show ${{ inputs.path }} --output .coverage-output 124 | cat .coverage-output 125 | shell: bash 126 | 127 | - name: Fetch report from ${{ inputs.diff-storage }} 128 | uses: actions/checkout@v4 129 | with: 130 | path: ${{ inputs.diff-storage }} 131 | fetch-depth: 0 132 | token: ${{ inputs.token }} 133 | 134 | - name: Get token identity 135 | id: identity 136 | uses: octokit/graphql-action@v2.x 137 | with: 138 | query: | 139 | query { 140 | viewer { 141 | databaseId 142 | login 143 | } 144 | } 145 | env: 146 | GITHUB_TOKEN: ${{ inputs.token }} 147 | 148 | - name: Configure git 149 | run: | 150 | name="${{ fromJSON(steps.identity.outputs.data).viewer.login }}" 151 | email="${{ format('{0}+{1}@users.noreply.github.com', fromJSON(steps.identity.outputs.data).viewer.databaseId, fromJSON(steps.identity.outputs.data).viewer.login) }}" 152 | 153 | cat >> "$GITHUB_ENV" << EOF 154 | GIT_AUTHOR_NAME=$name 155 | GIT_AUTHOR_EMAIL=$email 156 | GIT_COMMITTER_NAME=$name 157 | GIT_COMMITTER_EMAIL=$email 158 | EOF 159 | shell: bash 160 | 161 | - name: Initialize storage branch 162 | working-directory: ${{ inputs.diff-storage }} 163 | run: | 164 | # Switch to the branch if it already exists 165 | git switch ${{ inputs.diff-storage }} || true 166 | git pull origin ${{ inputs.diff-storage }} || true 167 | # Create the branch if it doesn't exist yet 168 | git checkout --orphan ${{ inputs.diff-storage }} || true 169 | # Ensure that the bare minimum components exist in the branch 170 | mkdir -p data 171 | touch README.md data/.gitkeep 172 | # Copy necessary files and folders to a temporary location 173 | mkdir -p /tmp/${{ github.sha }} 174 | echo "Copying data to /tmp/${{ github.sha }}" 175 | cp -r .git README.md data /tmp/${{ github.sha }} 176 | # Remove everything else 177 | # Attribution: https://unix.stackexchange.com/a/77313 178 | rm -rf ..?* .[!.]* * 179 | # Restore files from the temporary location 180 | echo "Copying data from /tmp/${{ github.sha }}" 181 | cp -r /tmp/${{ github.sha }}/.git /tmp/${{ github.sha }}/README.md /tmp/${{ github.sha }}/data . 182 | rm -rf /tmp/${{ github.sha }} 183 | git add --all -f 184 | git commit -m "Update storage branch: $(date)" || true 185 | shell: bash 186 | 187 | - name: Push storage branch 188 | uses: ad-m/github-push-action@master 189 | with: 190 | github_token: ${{ inputs.token }} 191 | branch: ${{ inputs.diff-storage }} 192 | directory: ${{ inputs.diff-storage }} 193 | force: true 194 | 195 | - name: Generate diff against ${{ inputs.diff-branch }} 196 | if: contains(inputs.diff, 'true') 197 | run: | 198 | echo "storage_subdirectory = '${{ inputs.storage-subdirectory }}'" 199 | pushd ${{ inputs.diff-storage }} 200 | git checkout ${{ inputs.diff-storage }} || touch ${{ inputs.diff-storage }}-not-found 201 | popd 202 | if [[ -f "${{ inputs.diff-storage }}/data/${{ inputs.diff-branch }}/${{ inputs.storage-subdirectory }}/coverage.xml" && (! -f ${{ inputs.diff-storage }}-not-found) ]] 203 | then { 204 | pycobertura diff --no-color --no-source ${{ inputs.diff-storage }}/data/${{ inputs.diff-branch }}/${{ inputs.storage-subdirectory }}/coverage.xml \ 205 | ${{ inputs.path }} \ 206 | --output .coverage-output.diff && pycobertura_status=$? || pycobertura_status=$? 207 | # Save status both in case of success and failure. 208 | echo "pycobertura_status=$pycobertura_status" >> $GITHUB_ENV 209 | cat .coverage-output.diff 210 | } else { 211 | echo "${{ inputs.diff-storage }}/data/${{ inputs.diff-branch }}/${{ inputs.storage-subdirectory }}/coverage.xml not found! Not diffing." 212 | } 213 | fi 214 | shell: bash 215 | 216 | - name: Extract diff-branch coverage percentage 217 | if: contains(inputs.diff, 'true') && contains(inputs.coverage-rate-reduction-failure, 'true') 218 | run: | 219 | if [[ -f "${{ inputs.diff-storage }}/data/${{ inputs.diff-branch }}/${{ inputs.storage-subdirectory }}/coverage.xml" ]] 220 | then { 221 | # Generate a text report for the diff-branch coverage 222 | pycobertura show ${{ inputs.diff-storage }}/data/${{ inputs.diff-branch }}/${{ inputs.storage-subdirectory }}/coverage.xml --output .coverage-output.diff-branch 223 | # Extract the total coverage percentage 224 | grep -E "^TOTAL " .coverage-output.diff-branch | \ 225 | awk '{printf "%.8f", (1 - $3/$2) * 100}' > .coverage-total.diff-branch 226 | echo "Diff-branch coverage: $(cat .coverage-total.diff-branch)%" 227 | } else { 228 | echo "Diff-branch coverage file not found, cannot compare coverage rates." 229 | echo "0" > .coverage-total.diff-branch 230 | } 231 | fi 232 | shell: bash 233 | 234 | - name: Get total 235 | run: | 236 | grep -E "^TOTAL " .coverage-output | \ 237 | awk '{printf "%.8f", (1 - $3/$2) * 100}' > .coverage-total 238 | shell: bash 239 | 240 | - name: Store coverage percent 241 | id: coverage_percent 242 | run: | 243 | echo "coverage_total=$(cat .coverage-total)" >> $GITHUB_OUTPUT 244 | BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} 245 | echo "diff_storage_branch=$BRANCH" >> $GITHUB_ENV 246 | mkdir -p ${{ inputs.diff-storage }}/data/${BRANCH}/${{ inputs.storage-subdirectory }} 247 | shell: bash 248 | 249 | # Use the output from the `coverage_percent` step 250 | - name: Generate the badge SVG image 251 | uses: emibcn/badge-action@v2.0.3 252 | id: badge 253 | with: 254 | label: 'Test Coverage' 255 | status: "${{ steps.coverage_percent.outputs.coverage_total }}%" 256 | color: ${{ 257 | steps.coverage_percent.outputs.coverage_total > 90 && 'green' || 258 | steps.coverage_percent.outputs.coverage_total > 80 && 'yellow,green' || 259 | steps.coverage_percent.outputs.coverage_total > 70 && 'yellow' || 260 | steps.coverage_percent.outputs.coverage_total > 60 && 'orange,yellow' || 261 | steps.coverage_percent.outputs.coverage_total > 50 && 'orange' || 262 | steps.coverage_percent.outputs.coverage_total > 40 && 'red,orange' || 263 | steps.coverage_percent.outputs.coverage_total > 30 && 'red,red,orange' || 264 | steps.coverage_percent.outputs.coverage_total > 20 && 'red,red,red,orange' || 265 | 'red' }} 266 | path: ${{ inputs.diff-storage }}/data/${{ env.diff_storage_branch }}/${{ inputs.storage-subdirectory }}/badge.svg 267 | 268 | - name: Commit badge 269 | working-directory: ${{ inputs.diff-storage }}/data 270 | run: | 271 | git switch ${{ inputs.diff-storage }} || true 272 | git pull origin ${{ inputs.diff-storage }} 273 | git add "${{ env.diff_storage_branch }}/${{ inputs.storage-subdirectory }}/badge.svg" 274 | git commit -m "Add/Update badge: ${{ github.sha }}" || true 275 | shell: bash 276 | 277 | # Badge has to be committed and pushed to be used in comment 278 | - name: Push badges 279 | uses: ad-m/github-push-action@master 280 | with: 281 | github_token: ${{ inputs.token }} 282 | branch: ${{ inputs.diff-storage }} 283 | directory: ${{ inputs.diff-storage }}/data 284 | 285 | - name: Determine repository visibility 286 | if: contains(inputs.publish, 'true') 287 | id: repository-visibility 288 | uses: actions/github-script@v7 289 | with: 290 | script: | 291 | const result = await github.rest.repos.get({ 292 | owner: "${{ github.repository_owner }}", 293 | repo: "${{ github.repository }}".split("/")[1] 294 | }); 295 | return result.data.visibility; 296 | result-encoding: string 297 | 298 | - name: Generate issue comment body 299 | if: contains(inputs.publish, 'true') 300 | run: | 301 | BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} 302 | if [[ "${{ steps.repository-visibility.outputs.result }}" == "public" ]] 303 | then { 304 | # URL encoding for branch name 305 | URL_ENCODED_BRANCH=$(python3 -c 'import urllib.parse; print(urllib.parse.quote_plus("${{ env.diff_storage_branch }}"))') 306 | echo -e "![badge](https://raw.githubusercontent.com/${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ inputs.diff-storage }}/data/${URL_ENCODED_BRANCH}/${{ inputs.storage-subdirectory }}/badge.svg)\n\n" > .coverage-output.final 307 | } 308 | else { 309 | echo -e "🧪 Test coverage: ${{ steps.coverage_percent.outputs.coverage_total }}%\n\n" > .coverage-output.final 310 | } 311 | fi 312 | echo -e "## ${{ inputs.coverage-summary-title }}\n" >> .coverage-output.final 313 | if [[ "${{ inputs.exclude-detailed-coverage }}" == "false" ]] 314 | then { 315 | if [[ "${{ inputs.togglable-report }}" == "true" ]] 316 | then { 317 | echo -e "
\n\n" >> .coverage-output.final 318 | } 319 | fi 320 | echo -e "\`\`\`" >> .coverage-output.final 321 | cat .coverage-output >> .coverage-output.final 322 | echo -e "\n\`\`\`\n" >> .coverage-output.final 323 | if [[ "${{ inputs.togglable-report }}" == "true" ]] 324 | then { 325 | echo -e "
\n\n" >> .coverage-output.final 326 | } 327 | fi 328 | } 329 | else { 330 | echo -e "\n" >> .coverage-output.final 331 | } 332 | fi 333 | if [[ "${{ inputs.diff }}" == "true" && -f .coverage-output.diff ]] 334 | then { 335 | echo -e "### Diff against ${{ inputs.diff-branch }}\n" >> .coverage-output.final 336 | echo -e "\`\`\`" >> .coverage-output.final 337 | cat .coverage-output.diff >> .coverage-output.final 338 | echo -e "\n\`\`\`\n" >> .coverage-output.final 339 | } 340 | fi 341 | COMMIT_SHA="${{ github.sha }}" 342 | if [ "${{ github.event_name }}" == "pull_request" ] 343 | then { 344 | COMMIT_SHA="${{ github.event.pull_request.head.sha }}" 345 | } 346 | fi 347 | echo -e "\nResults for commit: $COMMIT_SHA\n" >> .coverage-output.final 348 | echo -e "\n_Minimum allowed coverage is \`${{ inputs.threshold }}%\`_\n" >> .coverage-output.final 349 | if [[ "${{ inputs.publish }}" == "true" ]] 350 | then { 351 | echo -e "\n:recycle: This comment has been updated with latest results\n" >> .coverage-output.final 352 | } 353 | fi 354 | shell: bash 355 | 356 | - name: Post as comment 357 | if: contains(inputs.publish, 'true') 358 | uses: marocchino/sticky-pull-request-comment@v2 359 | with: 360 | GITHUB_TOKEN: ${{ inputs.token }} 361 | header: ${{ inputs.path }} 362 | path: .coverage-output.final 363 | 364 | - name: Set as output 365 | id: create-output 366 | run: | 367 | EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) 368 | echo "summary<<$EOF" >> $GITHUB_OUTPUT 369 | echo "$(cat .coverage-output.final)" >> $GITHUB_OUTPUT 370 | echo "$EOF" >> $GITHUB_OUTPUT 371 | shell: bash 372 | 373 | - name: Commit XML report to ${{ inputs.diff-storage }} 374 | if: > 375 | contains(inputs.diff, 'true') 376 | working-directory: ${{ inputs.diff-storage }} 377 | run: | 378 | git switch ${{ inputs.diff-storage }} 379 | git pull origin ${{ inputs.diff-storage }} 380 | filename=$(basename ${{ inputs.path }}) 381 | mv ../coverage-action/${filename} ./data/${{ env.diff_storage_branch }}/${{ inputs.storage-subdirectory }}/coverage.xml 382 | git add -f "./data/${{ env.diff_storage_branch }}/${{ inputs.storage-subdirectory }}/coverage.xml" 383 | git commit -m "Coverage report for ${{ github.sha }}" || true 384 | shell: bash 385 | 386 | - name: Push XML report to ${{ inputs.diff-storage }} 387 | if: > 388 | contains(inputs.diff, 'true') 389 | uses: ad-m/github-push-action@master 390 | with: 391 | github_token: ${{ inputs.token }} 392 | branch: ${{ inputs.diff-storage }} 393 | directory: ${{ inputs.diff-storage }}/data 394 | 395 | - name: Check threshold 396 | if: contains(inputs.fail, 'true') 397 | run: | 398 | with open('.coverage-total', 'r') as t: 399 | total = float(t.read().rstrip()) 400 | min = float('${{ inputs.threshold }}') 401 | if total < min: 402 | raise SystemExit( 403 | f"Total Coverage of {total}% falls below minimum threshold of {min}%." 404 | ) 405 | shell: python 406 | 407 | - name: Fail if coverage worsened 408 | if: contains(inputs.diff, 'true') 409 | run: | 410 | if [[ "${{ env.pycobertura_status }}" == "2" && "${{ inputs.uncovered-statements-increase-failure }}" == "true" ]] 411 | then { 412 | echo "Code changes increased the number of uncovered lines in at least one file." 413 | exit 1 414 | } 415 | fi 416 | if [[ "${{ env.pycobertura_status }}" == "3" && "${{ inputs.new-uncovered-statements-failure }}" == "true" ]] 417 | then { 418 | echo "Code changes introduced new uncovered statements despite overall coverage improvement." 419 | exit 1 420 | } 421 | fi 422 | if [[ "${{ env.pycobertura_status }}" == "1" && "${{ inputs.pycobertura-exception-failure }}" == "true" ]] 423 | then { 424 | echo "Pycobertura exception occurred." 425 | exit 1 426 | } 427 | fi 428 | if [[ "${{ inputs.coverage-rate-reduction-failure }}" == "true" && -f .coverage-total.diff-branch ]] 429 | then { 430 | current_coverage=$(cat .coverage-total) 431 | diff_branch_coverage=$(cat .coverage-total.diff-branch) 432 | if (( $(echo "$current_coverage < $diff_branch_coverage" | bc -l) )) 433 | then { 434 | echo "Coverage rate decreased from ${diff_branch_coverage}% to ${current_coverage}%." 435 | exit 1 436 | } 437 | fi 438 | } 439 | fi 440 | shell: bash 441 | 442 | - name: Clean up intermediate files 443 | if: always() 444 | run: | 445 | rm -rf coverage-action .coverage-output.final .coverage-output \ 446 | .coverage-total .coverage-output.diff \ 447 | .coverage-output.diff-branch .coverage-total.diff-branch \ 448 | ${{ inputs.diff-storage }}-not-found 449 | shell: bash 450 | -------------------------------------------------------------------------------- /example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/insightsengineering/coverage-action/50449af77a423ca017bf5496a6711db404d31811/example.png -------------------------------------------------------------------------------- /fixtures/README.md: -------------------------------------------------------------------------------- 1 | # Test Fixtures Documentation 2 | 3 | This directory contains test fixtures for the Code Coverage Report Action. Each fixture is designed to test specific scenarios and feature flags. 4 | 5 | ## Original Test Fixtures 6 | 7 | ### `test-branch.xml` 8 | - **Purpose**: Tests basic branch coverage functionality 9 | - **Coverage**: 90% line coverage, 75% branch coverage 10 | - **Features**: Contains multiple classes with varying coverage levels 11 | 12 | ### `test-missing-lines.xml` / `test-missing-lines-2.xml` 13 | - **Purpose**: Tests scenarios with missing lines in different patterns 14 | - **Coverage**: 51.25% line coverage 15 | - **Features**: Multiple files with different missing line patterns (start, end, gaps, etc.) 16 | 17 | ### `test-no-branch.xml` / `test-no-branch-2.xml` 18 | - **Purpose**: Tests scenarios without branch coverage data 19 | - **Coverage**: Varies by file 20 | - **Features**: Simple line coverage without branch information 21 | 22 | ### `test-python.xml` 23 | - **Purpose**: Tests Python-specific coverage format 24 | - **Coverage**: 90% line coverage 25 | - **Features**: Generated by coverage.py 26 | 27 | ## New Test Fixtures for Feature Flags 28 | 29 | ### Failure Mode Testing 30 | 31 | #### `test-uncovered-statements-increase.xml` 32 | - **Purpose**: Tests `uncovered-statements-increase-failure` flag 33 | - **Coverage**: 70% line coverage 34 | - **Scenario**: Represents increased uncovered statements (pycobertura exit code 2) 35 | - **Expected**: Should fail when flag is enabled 36 | 37 | #### `test-uncovered-statements-decreased.xml` 38 | - **Purpose**: Tests `uncovered-statements-increase-failure` flag (positive case) 39 | - **Coverage**: 90% line coverage 40 | - **Scenario**: Represents decreased uncovered statements 41 | - **Expected**: Should pass when flag is enabled 42 | 43 | #### `test-new-uncovered-statements.xml` 44 | - **Purpose**: Tests `new-uncovered-statements-failure` flag 45 | - **Coverage**: 80% line coverage 46 | - **Scenario**: Represents new uncovered statements with overall improvement (pycobertura exit code 3) 47 | - **Expected**: Should fail when flag is enabled 48 | 49 | #### `test-no-uncovered-statements.xml` 50 | - **Purpose**: Tests `new-uncovered-statements-failure` flag (positive case) 51 | - **Coverage**: 100% line coverage 52 | - **Scenario**: Represents no uncovered statements 53 | - **Expected**: Should pass when flag is enabled 54 | 55 | #### `test-coverage-rate-reduction.xml` 56 | - **Purpose**: Tests `coverage-rate-reduction-failure` flag 57 | - **Coverage**: 60% line coverage 58 | - **Scenario**: Represents reduced coverage rate compared to baseline 59 | - **Expected**: Should fail when flag is enabled 60 | 61 | #### `test-coverage-improved.xml` 62 | - **Purpose**: Tests `coverage-rate-reduction-failure` flag (positive case) 63 | - **Coverage**: 90% line coverage 64 | - **Scenario**: Represents improved coverage rate compared to baseline 65 | - **Expected**: Should pass when flag is enabled 66 | 67 | ### Exception Testing 68 | 69 | #### `test-pycobertura-exception.xml` 70 | - **Purpose**: Tests `pycobertura-exception-failure` flag in diff scenarios 71 | - **Coverage**: 80% line coverage 72 | - **Scenario**: Normal coverage report with diff enabled and exception flag enabled 73 | - **Expected**: Tests that the flag is properly handled during diff operations 74 | 75 | #### `test-exception-disabled.xml` 76 | - **Purpose**: Tests `pycobertura-exception-failure` flag (disabled case) 77 | - **Coverage**: 85% line coverage 78 | - **Scenario**: Normal coverage report 79 | - **Expected**: Should pass when flag is disabled 80 | 81 | ### Report Format Testing 82 | 83 | #### `test-togglable-report.xml` 84 | - **Purpose**: Tests `togglable-report` flag 85 | - **Coverage**: 85% line coverage 86 | - **Scenario**: Normal coverage report 87 | - **Expected**: Should generate collapsible report format when flag is enabled 88 | 89 | ### Comprehensive Testing 90 | 91 | #### `test-comprehensive.xml` 92 | - **Purpose**: Tests multiple feature flags together 93 | - **Coverage**: 75% line coverage across multiple files 94 | - **Scenario**: Contains three files with high, medium, and low coverage 95 | - **Features**: Tests complex scenarios with multiple files and varying coverage levels 96 | 97 | ### Baseline Testing 98 | 99 | #### `test-baseline-main.xml` 100 | - **Purpose**: Provides baseline coverage for diff testing 101 | - **Coverage**: 80% line coverage 102 | - **Scenario**: Represents the "main" branch coverage for comparison 103 | - **Features**: Used as reference point for diff-based failure mode testing 104 | 105 | ## Test Matrix Coverage 106 | 107 | The test matrix in `.github/workflows/test.yaml` includes test cases for: 108 | 109 | 1. **Basic functionality**: Original test fixtures with new flags set to defaults 110 | 2. **Individual failure modes**: Each failure flag tested in isolation 111 | 3. **Positive cases**: Scenarios where failure flags should NOT trigger 112 | 4. **Combined scenarios**: Multiple flags enabled together 113 | 5. **Exception handling**: Both enabled and disabled exception failure modes 114 | 6. **Report formatting**: Togglable reports with and without detailed coverage 115 | 7. **Edge cases**: Comprehensive testing with multiple files and varying coverage 116 | 117 | ## Usage 118 | 119 | Each test fixture is used in the GitHub Actions test matrix to verify that: 120 | 121 | - Feature flags work correctly in isolation 122 | - Feature flags work correctly when combined 123 | - Positive and negative scenarios are handled properly 124 | - Exception handling works as expected 125 | - Report formatting options function correctly 126 | 127 | The test fixtures are designed to be realistic and cover the various scenarios that users might encounter when using the coverage action with different configurations. -------------------------------------------------------------------------------- /fixtures/test-baseline-main.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /fixtures/test-branch.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | C:/local/mvn-coverage-example/src/main/java 7 | --source 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 121 | 122 | 123 | 124 | 125 | 126 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /fixtures/test-comprehensive.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /fixtures/test-coverage-improved.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /fixtures/test-coverage-rate-reduction.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /fixtures/test-exception-disabled.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /fixtures/test-missing-lines-2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /fixtures/test-missing-lines.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /fixtures/test-new-uncovered-statements.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /fixtures/test-no-branch-2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | C:/local/mvn-coverage-example/src/main/java 7 | --source 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 50 | 51 | 52 | 53 | 54 | 55 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /fixtures/test-no-branch.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | C:/local/mvn-coverage-example/src/main/java 7 | --source 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 50 | 51 | 52 | 53 | 54 | 55 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /fixtures/test-no-uncovered-statements.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /fixtures/test-pycobertura-exception.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /fixtures/test-python.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /fixtures/test-togglable-report.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /fixtures/test-uncovered-statements-decreased.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /fixtures/test-uncovered-statements-increase.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | --------------------------------------------------------------------------------