├── .Rbuildignore ├── .github ├── .gitignore ├── FUNDING.yml └── workflows │ ├── R-CMD-check.yaml │ ├── lint.yaml │ ├── pkgdown.yaml │ ├── pr-commands.yaml │ ├── rhub.yaml │ ├── test-acceptance.yaml │ ├── test-coverage.yaml │ └── test-mutation.yaml ├── .gitignore ├── .lintr ├── DESCRIPTION ├── LICENSE ├── LICENSE.md ├── NAMESPACE ├── NEWS.md ├── R ├── mutator-operator.R ├── muttest.R ├── project_copy_strategy.R ├── reporter-progress.R ├── reporter.R └── test_strategy.R ├── README.Rmd ├── README.md ├── _pkgdown.yml ├── codecov.yml ├── cran-comments.md ├── inst └── examples │ └── operators │ ├── DESCRIPTION │ ├── R │ └── calculate.R │ └── tests │ └── testthat │ ├── helper.R │ ├── setup.R │ └── test-calculate.R ├── man ├── CopyStrategy.Rd ├── FileTestStrategy.Rd ├── FullTestStrategy.Rd ├── MutationReporter.Rd ├── PackageCopyStrategy.Rd ├── ProgressMutationReporter.Rd ├── TestStrategy.Rd ├── default_copy_strategy.Rd ├── default_reporter.Rd ├── default_test_strategy.Rd ├── figures │ └── logo.png ├── muttest.Rd ├── operator.Rd └── plan.Rd ├── muttest.Rproj ├── pkgdown └── favicon │ ├── apple-touch-icon.png │ ├── favicon-96x96.png │ ├── favicon.ico │ ├── favicon.svg │ ├── site.webmanifest │ ├── web-app-manifest-192x192.png │ └── web-app-manifest-512x512.png ├── tests ├── acceptance │ ├── setup-steps.R │ ├── setup.R │ ├── test_box.feature │ ├── test_package.feature │ └── test_repo.feature ├── testthat.R └── testthat │ ├── test-acceptance.R │ ├── test-mutator-operator.R │ ├── test-muttest.R │ ├── test-project_copy_strategy.R │ └── test-test_strategy.R └── vignettes └── articles ├── .gitignore └── how-it-works.Rmd /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^.*\.Rproj$ 2 | ^\.Rproj\.user$ 3 | ^LICENSE\.md$ 4 | ^codecov\.yml$ 5 | ^_pkgdown\.yml$ 6 | ^docs$ 7 | ^pkgdown$ 8 | ^\.github$ 9 | ^README\.Rmd$ 10 | ^vignettes/articles$ 11 | ^\.lintr$ 12 | ^CRAN-SUBMISSION$ 13 | ^cran-comments\.md$ 14 | -------------------------------------------------------------------------------- /.github/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [jakubsob] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 12 | polar: # Replace with a single Polar username 13 | buy_me_a_coffee: # Replace with a single Buy Me a Coffee username 14 | thanks_dev: # Replace with a single thanks.dev username 15 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 16 | -------------------------------------------------------------------------------- /.github/workflows/R-CMD-check.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | 8 | name: R-CMD-check.yaml 9 | 10 | permissions: read-all 11 | 12 | jobs: 13 | R-CMD-check: 14 | runs-on: ${{ matrix.config.os }} 15 | 16 | name: ${{ matrix.config.os }} (${{ matrix.config.r }}) 17 | 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | config: 22 | - {os: macos-latest, r: 'release'} 23 | - {os: windows-latest, r: 'release'} 24 | - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} 25 | - {os: ubuntu-latest, r: 'release'} 26 | - {os: ubuntu-latest, r: 'oldrel-1'} 27 | 28 | env: 29 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 30 | R_KEEP_PKG_SOURCE: yes 31 | 32 | steps: 33 | - uses: actions/checkout@v4 34 | 35 | - uses: r-lib/actions/setup-pandoc@v2 36 | 37 | - uses: r-lib/actions/setup-r@v2 38 | with: 39 | r-version: ${{ matrix.config.r }} 40 | http-user-agent: ${{ matrix.config.http-user-agent }} 41 | use-public-rspm: true 42 | 43 | - uses: r-lib/actions/setup-r-dependencies@v2 44 | with: 45 | extra-packages: any::rcmdcheck 46 | needs: check 47 | 48 | - uses: r-lib/actions/check-r-package@v2 49 | with: 50 | upload-snapshots: true 51 | build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")' 52 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | 8 | name: lint.yaml 9 | 10 | permissions: read-all 11 | 12 | jobs: 13 | lint: 14 | runs-on: ubuntu-latest 15 | env: 16 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 17 | steps: 18 | - uses: actions/checkout@v4 19 | 20 | - uses: r-lib/actions/setup-r@v2 21 | with: 22 | use-public-rspm: true 23 | 24 | - uses: r-lib/actions/setup-r-dependencies@v2 25 | with: 26 | extra-packages: any::lintr, local::. 27 | needs: lint 28 | 29 | - name: Lint 30 | run: lintr::lint_package() 31 | shell: Rscript {0} 32 | env: 33 | LINTR_ERROR_ON_LINT: true 34 | -------------------------------------------------------------------------------- /.github/workflows/pkgdown.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | release: 8 | types: [published] 9 | workflow_dispatch: 10 | 11 | name: pkgdown.yaml 12 | 13 | permissions: read-all 14 | 15 | jobs: 16 | pkgdown: 17 | runs-on: ubuntu-latest 18 | # Only restrict concurrency for non-PR jobs 19 | concurrency: 20 | group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} 21 | env: 22 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 23 | permissions: 24 | contents: write 25 | steps: 26 | - uses: actions/checkout@v4 27 | 28 | - uses: r-lib/actions/setup-pandoc@v2 29 | 30 | - uses: r-lib/actions/setup-r@v2 31 | with: 32 | use-public-rspm: true 33 | 34 | - uses: r-lib/actions/setup-r-dependencies@v2 35 | with: 36 | extra-packages: any::pkgdown, local::. 37 | needs: website 38 | 39 | - name: Build site 40 | run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) 41 | shell: Rscript {0} 42 | 43 | - name: Deploy to GitHub pages 🚀 44 | if: github.event_name != 'pull_request' 45 | uses: JamesIves/github-pages-deploy-action@v4.5.0 46 | with: 47 | clean: false 48 | branch: gh-pages 49 | folder: docs 50 | -------------------------------------------------------------------------------- /.github/workflows/pr-commands.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | issue_comment: 5 | types: [created] 6 | 7 | name: pr-commands.yaml 8 | 9 | permissions: read-all 10 | 11 | jobs: 12 | document: 13 | if: ${{ github.event.issue.pull_request && (github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER') && startsWith(github.event.comment.body, '/document') }} 14 | name: document 15 | runs-on: ubuntu-latest 16 | env: 17 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 18 | permissions: 19 | contents: write 20 | steps: 21 | - uses: actions/checkout@v4 22 | 23 | - uses: r-lib/actions/pr-fetch@v2 24 | with: 25 | repo-token: ${{ secrets.GITHUB_TOKEN }} 26 | 27 | - uses: r-lib/actions/setup-r@v2 28 | with: 29 | use-public-rspm: true 30 | 31 | - uses: r-lib/actions/setup-r-dependencies@v2 32 | with: 33 | extra-packages: any::roxygen2 34 | needs: pr-document 35 | 36 | - name: Document 37 | run: roxygen2::roxygenise() 38 | shell: Rscript {0} 39 | 40 | - name: commit 41 | run: | 42 | git config --local user.name "$GITHUB_ACTOR" 43 | git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com" 44 | git add man/\* NAMESPACE 45 | git commit -m 'Document' 46 | 47 | - uses: r-lib/actions/pr-push@v2 48 | with: 49 | repo-token: ${{ secrets.GITHUB_TOKEN }} 50 | 51 | style: 52 | if: ${{ github.event.issue.pull_request && (github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER') && startsWith(github.event.comment.body, '/style') }} 53 | name: style 54 | runs-on: ubuntu-latest 55 | env: 56 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 57 | permissions: 58 | contents: write 59 | steps: 60 | - uses: actions/checkout@v4 61 | 62 | - uses: r-lib/actions/pr-fetch@v2 63 | with: 64 | repo-token: ${{ secrets.GITHUB_TOKEN }} 65 | 66 | - uses: r-lib/actions/setup-r@v2 67 | 68 | - name: Install dependencies 69 | run: install.packages("styler") 70 | shell: Rscript {0} 71 | 72 | - name: Style 73 | run: styler::style_pkg() 74 | shell: Rscript {0} 75 | 76 | - name: commit 77 | run: | 78 | git config --local user.name "$GITHUB_ACTOR" 79 | git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com" 80 | git add \*.R 81 | git commit -m 'Style' 82 | 83 | - uses: r-lib/actions/pr-push@v2 84 | with: 85 | repo-token: ${{ secrets.GITHUB_TOKEN }} 86 | -------------------------------------------------------------------------------- /.github/workflows/rhub.yaml: -------------------------------------------------------------------------------- 1 | # R-hub's generic GitHub Actions workflow file. It's canonical location is at 2 | # https://github.com/r-hub/actions/blob/v1/workflows/rhub.yaml 3 | # You can update this file to a newer version using the rhub2 package: 4 | # 5 | # rhub::rhub_setup() 6 | # 7 | # It is unlikely that you need to modify this file manually. 8 | 9 | name: R-hub 10 | run-name: "${{ github.event.inputs.id }}: ${{ github.event.inputs.name || format('Manually run by {0}', github.triggering_actor) }}" 11 | 12 | on: 13 | workflow_dispatch: 14 | inputs: 15 | config: 16 | description: 'A comma separated list of R-hub platforms to use.' 17 | type: string 18 | default: 'linux,windows,macos' 19 | name: 20 | description: 'Run name. You can leave this empty now.' 21 | type: string 22 | id: 23 | description: 'Unique ID. You can leave this empty now.' 24 | type: string 25 | 26 | jobs: 27 | 28 | setup: 29 | runs-on: ubuntu-latest 30 | outputs: 31 | containers: ${{ steps.rhub-setup.outputs.containers }} 32 | platforms: ${{ steps.rhub-setup.outputs.platforms }} 33 | 34 | steps: 35 | # NO NEED TO CHECKOUT HERE 36 | - uses: r-hub/actions/setup@v1 37 | with: 38 | config: ${{ github.event.inputs.config }} 39 | id: rhub-setup 40 | 41 | linux-containers: 42 | needs: setup 43 | if: ${{ needs.setup.outputs.containers != '[]' }} 44 | runs-on: ubuntu-latest 45 | name: ${{ matrix.config.label }} 46 | strategy: 47 | fail-fast: false 48 | matrix: 49 | config: ${{ fromJson(needs.setup.outputs.containers) }} 50 | container: 51 | image: ${{ matrix.config.container }} 52 | 53 | steps: 54 | - uses: r-hub/actions/checkout@v1 55 | - uses: r-hub/actions/platform-info@v1 56 | with: 57 | token: ${{ secrets.RHUB_TOKEN }} 58 | job-config: ${{ matrix.config.job-config }} 59 | - uses: r-hub/actions/setup-deps@v1 60 | with: 61 | token: ${{ secrets.RHUB_TOKEN }} 62 | job-config: ${{ matrix.config.job-config }} 63 | - uses: r-hub/actions/run-check@v1 64 | with: 65 | token: ${{ secrets.RHUB_TOKEN }} 66 | job-config: ${{ matrix.config.job-config }} 67 | 68 | other-platforms: 69 | needs: setup 70 | if: ${{ needs.setup.outputs.platforms != '[]' }} 71 | runs-on: ${{ matrix.config.os }} 72 | name: ${{ matrix.config.label }} 73 | strategy: 74 | fail-fast: false 75 | matrix: 76 | config: ${{ fromJson(needs.setup.outputs.platforms) }} 77 | 78 | steps: 79 | - uses: r-hub/actions/checkout@v1 80 | - uses: r-hub/actions/setup-r@v1 81 | with: 82 | job-config: ${{ matrix.config.job-config }} 83 | token: ${{ secrets.RHUB_TOKEN }} 84 | - uses: r-hub/actions/platform-info@v1 85 | with: 86 | token: ${{ secrets.RHUB_TOKEN }} 87 | job-config: ${{ matrix.config.job-config }} 88 | - uses: r-hub/actions/setup-deps@v1 89 | with: 90 | job-config: ${{ matrix.config.job-config }} 91 | token: ${{ secrets.RHUB_TOKEN }} 92 | - uses: r-hub/actions/run-check@v1 93 | with: 94 | job-config: ${{ matrix.config.job-config }} 95 | token: ${{ secrets.RHUB_TOKEN }} 96 | -------------------------------------------------------------------------------- /.github/workflows/test-acceptance.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | 8 | name: test-acceptance.yaml 9 | 10 | permissions: read-all 11 | 12 | jobs: 13 | test-coverage: 14 | runs-on: ubuntu-latest 15 | env: 16 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - uses: r-lib/actions/setup-r@v2 22 | with: 23 | use-public-rspm: true 24 | 25 | - uses: r-lib/actions/setup-r-dependencies@v2 26 | 27 | - name: Test acceptance 28 | run: | 29 | pkgload::load_all() 30 | cucumber::test() 31 | shell: Rscript {0} 32 | -------------------------------------------------------------------------------- /.github/workflows/test-coverage.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | 8 | name: test-coverage.yaml 9 | 10 | permissions: read-all 11 | 12 | jobs: 13 | test-coverage: 14 | runs-on: ubuntu-latest 15 | env: 16 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - uses: r-lib/actions/setup-r@v2 22 | with: 23 | use-public-rspm: true 24 | 25 | - uses: r-lib/actions/setup-r-dependencies@v2 26 | with: 27 | extra-packages: any::covr, any::xml2 28 | needs: coverage 29 | 30 | - name: Test coverage 31 | run: | 32 | cov <- covr::package_coverage( 33 | quiet = FALSE, 34 | clean = FALSE, 35 | install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package") 36 | ) 37 | print(cov) 38 | covr::to_cobertura(cov) 39 | shell: Rscript {0} 40 | 41 | - uses: codecov/codecov-action@v5 42 | with: 43 | # Fail if error if not on PR, or if on PR and token is given 44 | fail_ci_if_error: ${{ github.event_name != 'pull_request' || secrets.CODECOV_TOKEN }} 45 | files: ./cobertura.xml 46 | plugins: noop 47 | disable_search: true 48 | token: ${{ secrets.CODECOV_TOKEN }} 49 | 50 | - name: Show testthat output 51 | if: always() 52 | run: | 53 | ## -------------------------------------------------------------------- 54 | find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true 55 | shell: bash 56 | 57 | - name: Upload test results 58 | if: failure() 59 | uses: actions/upload-artifact@v4 60 | with: 61 | name: coverage-test-failures 62 | path: ${{ runner.temp }}/package 63 | -------------------------------------------------------------------------------- /.github/workflows/test-mutation.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | 6 | name: test-mutation.yml 7 | permissions: write-all 8 | 9 | jobs: 10 | generate-badge: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v3 15 | with: 16 | fetch-depth: 0 17 | 18 | - uses: r-lib/actions/setup-r@v2 19 | with: 20 | use-public-rspm: true 21 | 22 | - uses: r-lib/actions/setup-r-dependencies@v2 23 | with: 24 | extra-packages: any::jsonlite 25 | 26 | - name: Set up Git user 27 | run: | 28 | git config user.name "github-actions" 29 | git config user.email "github-actions@github.com" 30 | 31 | - name: Update badge on badges branch to show "NA" 32 | run: | 33 | # Save current branch name 34 | CURRENT_BRANCH=$(git symbolic-ref --short HEAD) 35 | 36 | # Try to fetch the badges branch 37 | git fetch origin badges || echo "No badges branch yet, will create it" 38 | 39 | # If badges branch exists remotely, check it out 40 | if git show-ref --verify --quiet refs/remotes/origin/badges; then 41 | echo "Remote badges branch exists, checking it out" 42 | git checkout -b badges origin/badges || git checkout badges 43 | else 44 | # Create a new orphan branch 45 | echo "Creating new badges branch" 46 | git checkout --orphan badges 47 | # Remove all files from staging 48 | git rm -rf --cached . 49 | # Create empty commit to start the branch 50 | git commit --allow-empty -m "Initial empty commit for badges branch" 51 | fi 52 | 53 | # Create badges directory 54 | mkdir -p .badges 55 | 56 | # Create temporary NA badge 57 | echo '{ 58 | "schemaVersion": 1, 59 | "label": "muttest", 60 | "labelColor": "#0f2a13", 61 | "message": "NA", 62 | "color": "#858585", 63 | "logoSvg": "" 64 | }' > .badges/muttest.json 65 | 66 | # Add, commit and push 67 | git add .badges/muttest.json 68 | git commit -m "chore: :gears: Set badge message to NA before testing" || echo "No changes to commit" 69 | git push --force-with-lease origin badges 70 | 71 | # Return to the original branch 72 | git checkout $CURRENT_BRANCH 73 | 74 | - name: Mutation testing 75 | shell: Rscript {0} 76 | run: | 77 | dir.create(".badges", showWarnings = FALSE) 78 | color_scale <- function(score) { 79 | rgb <- round(colorRamp(c("#D61F1F", "#FFD301", "#006B3D"))(score)) 80 | sprintf("#%02X%02X%02X", rgb[1], rgb[2], rgb[3]) 81 | } 82 | 83 | create_badge <- function(score) { 84 | jsonlite::write_json( 85 | list( 86 | schemaVersion = 1, 87 | label = "muttest", 88 | labelColor = "#0f2a13", 89 | message = paste0(round(score, digits = 2) * 100, "%"), 90 | color = color_scale(score), 91 | logoSvg = "" 92 | ), 93 | path = ".badges/muttest.json", 94 | auto_unbox = TRUE 95 | ) 96 | } 97 | 98 | pkgload::load_all() 99 | 100 | plan <- plan( 101 | mutators = list( 102 | operator("+", "-"), 103 | operator("-", "+"), 104 | operator("*", "/"), 105 | operator("/", "*"), 106 | operator("==", "!="), 107 | operator("!=", "=="), 108 | operator("<", ">"), 109 | operator(">", "<"), 110 | operator("<=", ">="), 111 | operator(">=", "<=") 112 | ) 113 | ) 114 | score <- muttest(plan) 115 | 116 | create_badge(score) 117 | 118 | - name: Update badge with test results 119 | run: | 120 | # Save the badge file to a temporary location 121 | mkdir -p /tmp/muttest-badge 122 | mv .badges/muttest.json /tmp/muttest-badge/ 123 | 124 | # Fetch and checkout badges branch 125 | git fetch origin badges 126 | git checkout badges 127 | 128 | # Copy the badge file from temp location to badges directory 129 | cp /tmp/muttest-badge/muttest.json .badges/ 130 | 131 | # Add, commit and push 132 | git add .badges/muttest.json 133 | git commit -m "chore: :gears: Update muttest badge with test results" || echo "No changes to commit" 134 | git push --force-with-lease origin badges 135 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | .Renviron 6 | docs 7 | -------------------------------------------------------------------------------- /.lintr: -------------------------------------------------------------------------------- 1 | linters: linters_with_defaults( 2 | line_length_linter = line_length_linter(120), 3 | object_name_linter = object_name_linter(styles = c("snake_case", "CamelCase")) 4 | ) 5 | encoding: "UTF-8" 6 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: muttest 2 | Type: Package 3 | Title: Mutation Testing 4 | Version: 0.1.0 5 | Authors@R: c( 6 | person( 7 | "Jakub", "Sobolewski", 8 | email = "jakupsob@gmail.com", 9 | role = c("aut", "cre") 10 | ) 11 | ) 12 | Description: Measure quality of your tests. 13 | 'muttest' introduces small changes (mutations) to your code 14 | and runs your tests to check if they catch the changes. 15 | If they do, your tests are good. 16 | If not, your assertions are not specific enough. 17 | 'muttest' gives you percent score of how often your tests catch the changes. 18 | License: MIT + file LICENSE 19 | Encoding: UTF-8 20 | RoxygenNote: 7.3.2 21 | Depends: 22 | R (>= 4.1.0) 23 | Imports: 24 | checkmate, 25 | cli, 26 | dplyr, 27 | fs, 28 | purrr, 29 | R6, 30 | rlang, 31 | testthat, 32 | tibble, 33 | treesitter, 34 | treesitter.r, 35 | withr 36 | Config/testthat/edition: 3 37 | URL: https://jakubsob.github.io/muttest/ 38 | Suggests: 39 | box, 40 | covr, 41 | cucumber (>= 2.1.0), 42 | ggplot2, 43 | shiny 44 | Config/Needs/website: rmarkdown 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | YEAR: 2025 2 | COPYRIGHT HOLDER: muttest authors 3 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # MIT License 2 | 3 | Copyright (c) 2025 muttest authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | export(CopyStrategy) 4 | export(FileTestStrategy) 5 | export(FullTestStrategy) 6 | export(MutationReporter) 7 | export(PackageCopyStrategy) 8 | export(ProgressMutationReporter) 9 | export(TestStrategy) 10 | export(default_copy_strategy) 11 | export(default_reporter) 12 | export(default_test_strategy) 13 | export(muttest) 14 | export(operator) 15 | export(plan) 16 | import(testthat) 17 | importFrom(R6,R6Class) 18 | importFrom(cli,col_green) 19 | importFrom(cli,col_grey) 20 | importFrom(cli,col_red) 21 | importFrom(cli,col_yellow) 22 | importFrom(cli,symbol) 23 | importFrom(rlang,.data) 24 | importFrom(rlang,`%||%`) 25 | -------------------------------------------------------------------------------- /NEWS.md: -------------------------------------------------------------------------------- 1 | # 0.1.0 2 | 3 | - ✨ Create a testing plan with `plan`. 4 | - ✨ Run mutation testing with `muttest`. 5 | - ✨ Support mutating operators with `operator`. 6 | - ✨ Control copying project to temporary directory with `CopyStrategy`: 7 | - `PackageCopyStrategy` implemented for copying package files. 8 | - ✨ Control test execution for each mutant with `TestStrategy`: 9 | - `FullTestStrategy` for running all tests for each mutant. 10 | - `FileTestStrategy` for running only test files matching mutant files. 11 | - ✨ See test results with `MutationReporter`. 12 | - `ProgressMutationReporter` for printing progress to the console. 13 | -------------------------------------------------------------------------------- /R/mutator-operator.R: -------------------------------------------------------------------------------- 1 | Mutator <- R6::R6Class( 2 | public = list( 3 | from = NULL, 4 | to = NULL, 5 | query = NULL, 6 | initialize = function(from, to, query) { 7 | self$from <- from 8 | self$to <- to 9 | self$query <- query 10 | }, 11 | mutate = function(code) { 12 | mutate_code(code, self) 13 | }, 14 | # nocov start 15 | print = function() { 16 | cat(sprintf("Mutator: %s -> %s\n", self$from, self$to)) 17 | cat(sprintf("Query: %s\n", self$query)) 18 | } 19 | # nocov end 20 | ) 21 | ) 22 | 23 | #' Mutate an operator 24 | #' 25 | #' It changes a binary operator to another one. 26 | #' 27 | #' @examples 28 | #' operator("==", "!=") 29 | #' operator(">", "<") 30 | #' operator("<", ">") 31 | #' operator("+", "-") 32 | #' 33 | #' @param from The operator to be replaced. 34 | #' @param to The operator to replace with. 35 | #' @export 36 | operator <- function(from, to) { 37 | Mutator$new( 38 | from = from, 39 | to = to, 40 | query = sprintf('(binary_operator 41 | lhs: (_) @lhs 42 | operator: _ @operator 43 | rhs: (_) @rhs 44 | (#eq? @operator "%s") 45 | )', from) 46 | ) 47 | } 48 | 49 | info_oneline <- function(m) { 50 | paste(m$from, cli::symbol$arrow_right, m$to) 51 | } 52 | 53 | replace <- function(code, node, mutator) { 54 | start_point <- treesitter::node_start_point(node) 55 | code[start_point$row + 1] <- paste0( 56 | substr(code[start_point$row + 1], 1, start_point$column), 57 | mutator$to, 58 | substr( 59 | code[start_point$row + 1], 60 | start_point$column + nchar(mutator$from) + 1, 61 | nchar(code[start_point$row + 1]) 62 | ) 63 | ) 64 | code 65 | } 66 | 67 | mutate_code <- function(code, mutator) { 68 | language <- treesitter.r::language() 69 | parser <- treesitter::parser(language) 70 | treesitter_code <- paste(code, collapse = "\n") 71 | tree <- treesitter::parser_parse(parser, treesitter_code) 72 | root_node <- treesitter::tree_root_node(tree) 73 | 74 | query <- treesitter::query(language, mutator$query) 75 | captures <- treesitter::query_captures(query, root_node) 76 | 77 | if (length(captures$node) == 0) { 78 | return(NULL) 79 | } 80 | 81 | mutations <- list() 82 | for (i in seq_along(captures$node)) { 83 | node <- captures$node[[i]] 84 | if (treesitter::node_text(node) != mutator$from) { 85 | next 86 | } 87 | mutations <- append( 88 | mutations, 89 | list( 90 | replace(code, node, mutator) 91 | ) 92 | ) 93 | } 94 | 95 | mutations 96 | } 97 | -------------------------------------------------------------------------------- /R/muttest.R: -------------------------------------------------------------------------------- 1 | #' Run a mutation test 2 | #' 3 | #' @param plan A data frame with the test plan. See `plan()`. 4 | #' @param path Path to the test directory. 5 | #' @param reporter Reporter to use for mutation testing results. See `?MutationReporter`. 6 | #' @param test_strategy Strategy for running tests. See `?TestStrategy`. 7 | #' The purpose of test strategy is to control how tests are executed. 8 | #' We can run all tests for each mutant, or only tests that are relevant to the mutant. 9 | #' @param copy_strategy Strategy for copying the project. See `?CopyStrategy`. 10 | #' This strategy controls which files are copied to the temporary directory, where the tests are run. 11 | #' 12 | #' @return A numeric value representing the mutation score. 13 | #' 14 | #' @export 15 | #' @md 16 | #' @importFrom rlang .data 17 | muttest <- function( 18 | plan, 19 | path = "tests/testthat", 20 | reporter = default_reporter(), 21 | test_strategy = default_test_strategy(), 22 | copy_strategy = default_copy_strategy() 23 | ) { 24 | checkmate::assert_directory_exists(path) 25 | checkmate::assert( 26 | checkmate::check_data_frame(plan), 27 | checkmate::check_set_equal( 28 | c("filename", "original_code", "mutated_code", "mutator"), 29 | names(plan) 30 | ), 31 | combine = "and" 32 | ) 33 | checkmate::assert_class(reporter, "MutationReporter") 34 | checkmate::assert_class(test_strategy, "TestStrategy", null.ok = TRUE) 35 | checkmate::assert_class(copy_strategy, "CopyStrategy") 36 | 37 | if (nrow(plan) == 0) { 38 | return(invisible(NA_real_)) 39 | } 40 | 41 | reporter$start_reporter(plan) 42 | 43 | plan |> 44 | dplyr::arrange(.data$filename, .data$mutator) |> 45 | dplyr::rowwise() |> 46 | dplyr::group_split() |> 47 | purrr::walk(\(row) { 48 | mutator <- row$mutator[[1]] 49 | filename <- row$filename 50 | mutated_code <- row$mutated_code[[1]] 51 | 52 | reporter$start_file(filename) 53 | reporter$start_mutator(mutator) 54 | reporter$update(force = TRUE) 55 | 56 | dir <- copy_strategy$execute(getwd(), row) 57 | checkmate::assert_directory_exists(dir) 58 | on.exit(fs::dir_delete(dir)) 59 | withr::with_tempdir(tmpdir = dir, pattern = "", { 60 | withr::with_dir(dir, { 61 | temp_filename <- file.path(dir, filename) 62 | writeLines(mutated_code, temp_filename) 63 | 64 | test_results <- test_strategy$execute( 65 | path = path, 66 | plan = row, 67 | reporter = reporter$test_reporter 68 | ) 69 | checkmate::assert_class(test_results, "testthat_results") 70 | }) 71 | }) 72 | 73 | test_results_tibble <- tibble::as_tibble(test_results) 74 | killed <- as.numeric(sum(test_results_tibble$failed) > 0) 75 | survived <- as.numeric(sum(test_results_tibble$failed) == 0) 76 | errors <- sum(test_results_tibble$error) 77 | 78 | reporter$add_result( 79 | row, 80 | killed, 81 | survived, 82 | errors 83 | ) 84 | reporter$end_mutator() 85 | reporter$end_file() 86 | }) 87 | 88 | reporter$end_reporter() 89 | invisible(reporter$get_score()) 90 | } 91 | 92 | #' Create a plan for mutation testing 93 | #' 94 | #' Each mutant requires rerunning the tests. For large project it might be not feasible to test all 95 | #' mutants in one go. This function allows you to create a plan for selected source files and mutators. 96 | #' 97 | #' The plan is in a data frame format, where each row represents a mutant. 98 | #' 99 | #' You can subset the plan before passing it to the `muttest()` function. 100 | #' 101 | #' @param mutators A list of mutators to use. See [operator()]. 102 | #' @param source_files A vector of file paths to the source files. 103 | #' @return A data frame with the test plan. 104 | #' The data frame has the following columns: 105 | #' - `filename`: The name of the source file. 106 | #' - `original_code`: The original code of the source file. 107 | #' - `mutated_code`: The mutated code of the source file. 108 | #' - `mutator`: The mutator that was applied. 109 | #' 110 | #' @export 111 | #' @md 112 | plan <- function( 113 | mutators, 114 | source_files = fs::dir_ls("R", regexp = ".[rR]$") 115 | ) { 116 | checkmate::assert_file_exists(source_files, extension = c("R", "r")) 117 | checkmate::assert_list(mutators) 118 | map_dfr <- purrr::compose(dplyr::bind_rows, purrr::map) 119 | map_dfr(mutators, function(mutator) { 120 | map_dfr(source_files, function(filename) { 121 | code_lines <- readLines(filename) 122 | mutations <- mutator$mutate(code_lines) 123 | if (length(mutations) == 0) { 124 | return( 125 | tibble::tibble( 126 | filename = character(), 127 | original_code = list(character()), 128 | mutated_code = list(character()), 129 | mutator = list(mutator) 130 | ) 131 | ) 132 | } 133 | map_dfr(mutations, function(mutation) { 134 | tibble::tibble( 135 | filename = filename, 136 | original_code = list(code_lines), 137 | mutated_code = list(mutation), 138 | mutator = list(mutator) 139 | ) 140 | }) 141 | }) 142 | }) 143 | } 144 | -------------------------------------------------------------------------------- /R/project_copy_strategy.R: -------------------------------------------------------------------------------- 1 | #' @title CopyStrategy interface 2 | #' 3 | #' @description 4 | #' Extend this class to implement a custom copy strategy. 5 | #' 6 | #' @md 7 | #' @export 8 | #' @family CopyStrategy 9 | CopyStrategy <- R6::R6Class( 10 | classname = "CopyStrategy", 11 | public = list( 12 | #' @description 13 | #' Copy project files according to the strategy 14 | #' 15 | #' @param original_dir The original directory to copy from 16 | #' @param plan The current test plan 17 | #' @return The path to the temporary directory 18 | execute = function(original_dir) { 19 | rlang::abort("Not implemented") 20 | } 21 | ) 22 | ) 23 | 24 | #' @title Package copy strategy 25 | #' 26 | #' @description 27 | #' It copies all files and directories from the original directory to a temporary directory. 28 | #' 29 | #' @md 30 | #' @export 31 | #' @family CopyStrategy 32 | PackageCopyStrategy <- R6::R6Class( 33 | classname = "PackageCopyStrategy", 34 | inherit = CopyStrategy, 35 | public = list( 36 | #' @description 37 | #' Copy project files, excluding hidden and temp directories 38 | #' 39 | #' @param original_dir The original directory to copy from 40 | #' @param plan The current test plan 41 | #' @return The path to the temporary directory 42 | execute = function(original_dir, plan) { 43 | temp_dir <- fs::path(tempdir(), digest::digest(plan)) 44 | 45 | dirs_to_copy <- list.dirs( 46 | original_dir, 47 | recursive = FALSE, 48 | full.names = FALSE 49 | ) 50 | dirs_to_copy <- dirs_to_copy[!grepl("^\\.|tmp|temp", dirs_to_copy)] 51 | 52 | purrr::walk(dirs_to_copy, function(dir) { 53 | src_path <- file.path(original_dir, dir) 54 | if (dir.exists(src_path)) { 55 | fs::dir_copy( 56 | src_path, 57 | file.path(temp_dir, dir), 58 | overwrite = TRUE 59 | ) 60 | } 61 | }) 62 | 63 | files <- fs::dir_ls(original_dir, type = "file") 64 | files <- fs::path_rel(files, original_dir) 65 | purrr::walk(files, function(x) { 66 | fs::file_copy( 67 | x, 68 | file.path(temp_dir, x), 69 | overwrite = TRUE 70 | ) 71 | }) 72 | 73 | temp_dir 74 | } 75 | ) 76 | ) 77 | 78 | #' Create a default project copy strategy 79 | #' 80 | #' @param ... Arguments passed to the `?PackageCopyStrategy` constructor. 81 | #' @return A `?CopyStrategy` object 82 | #' @md 83 | #' @export 84 | #' @family CopyStrategy 85 | default_copy_strategy <- function(...) { 86 | PackageCopyStrategy$new(...) 87 | } 88 | -------------------------------------------------------------------------------- /R/reporter-progress.R: -------------------------------------------------------------------------------- 1 | #' @title Progress Reporter for Mutation Testing 2 | #' 3 | #' @description 4 | #' A reporter that displays a progress indicator for mutation tests. 5 | #' It provides real-time feedback on which mutants are being tested and whether they were killed by tests. 6 | #' 7 | #' @field start_time Time when testing started (for duration calculation) 8 | #' @field min_time Minimum test duration to display timing information 9 | #' @field col_config List of column configuration for report formatting 10 | #' 11 | #' @importFrom R6 R6Class 12 | #' @importFrom cli col_green col_red col_yellow col_grey symbol 13 | #' @md 14 | #' @export 15 | #' @family MutationReporter 16 | ProgressMutationReporter <- R6::R6Class( 17 | classname = "ProgressMutationReporter", 18 | inherit = MutationReporter, 19 | public = list( 20 | start_time = NULL, 21 | min_time = 1, 22 | col_config = list( 23 | "status" = list( 24 | padding_left = 0, 25 | padding_right = 1, 26 | width = 2 27 | ), 28 | "k" = list( 29 | padding_left = 1, 30 | padding_right = 1, 31 | width = 5, 32 | type = "number" 33 | ), 34 | "s" = list( 35 | padding_left = 1, 36 | padding_right = 1, 37 | width = 5, 38 | type = "number" 39 | ), 40 | "e" = list( 41 | padding_left = 1, 42 | padding_right = 1, 43 | width = 5, 44 | type = "number" 45 | ), 46 | "t" = list( 47 | padding_left = 1, 48 | padding_right = 1, 49 | width = 5, 50 | type = "number" 51 | ), 52 | "score" = list( 53 | padding_left = 1, 54 | padding_right = 1, 55 | width = 5, 56 | type = "number" 57 | ), 58 | "mutator" = list( 59 | padding_left = 1, 60 | padding_right = 1, 61 | width = 10 62 | ), 63 | "file" = list( 64 | padding_left = 1, 65 | padding_right = 1 66 | ) 67 | ), 68 | 69 | #' @description Format a column with specified padding and width 70 | #' @param text Text to format 71 | #' @param col_name Column name to use configuration from 72 | #' @param colorize Optional function to color the text 73 | format_column = function(text, col_name, colorize = NULL) { 74 | config <- self$col_config[[col_name]] 75 | 76 | # Get actual visible length of text 77 | text_len <- nchar(text) 78 | 79 | # If width is specified, calculate available text space after padding 80 | if (!is.null(config$width) && config$width > 0) { 81 | padding_size <- (config$padding_left %||% 0) + 82 | (config$padding_right %||% 0) 83 | text_max_width <- config$width - padding_size 84 | if (text_len > text_max_width && text_max_width > 0) { 85 | # Truncate if text is too long for the available space 86 | text <- substring(text, 1, text_max_width) 87 | text_len <- text_max_width 88 | } 89 | } else if (!is.null(config$max_width) && config$max_width > 0) { 90 | # Otherwise, use the existing max_width logic if provided 91 | text <- substring(text, 1, config$max_width) 92 | text_len <- min(text_len, config$max_width) 93 | } 94 | 95 | # Calculate left padding 96 | left_pad <- "" 97 | if (!is.null(config$padding_left) && config$padding_left > 0) { 98 | left_pad <- strrep(" ", config$padding_left) 99 | } 100 | 101 | # Calculate right padding 102 | right_pad <- "" 103 | if (!is.null(config$padding_right) && config$padding_right > 0) { 104 | right_pad <- strrep(" ", config$padding_right) 105 | } 106 | 107 | # Handle fixed width columns 108 | if (!is.null(config$width) && config$width > 0) { 109 | # Calculate current total width 110 | total_width <- text_len + nchar(left_pad) + nchar(right_pad) 111 | if (total_width < config$width) { 112 | # For number columns, add extra spaces to left padding for right alignment 113 | if (!is.null(config$type) && config$type == "number") { 114 | left_pad <- paste0( 115 | left_pad, 116 | strrep(" ", config$width - total_width) 117 | ) 118 | } else { 119 | # Otherwise add to right padding (left alignment) 120 | right_pad <- paste0( 121 | right_pad, 122 | strrep(" ", config$width - total_width) 123 | ) 124 | } 125 | } 126 | } else if (!is.null(config$max_width) && config$max_width > 0) { 127 | # Handle max_width padding (existing behavior) 128 | total_current_width <- text_len + nchar(left_pad) + nchar(right_pad) 129 | if (total_current_width < config$max_width) { 130 | # Add extra spaces to reach max width 131 | right_pad <- paste0( 132 | right_pad, 133 | strrep(" ", config$max_width - total_current_width) 134 | ) 135 | } 136 | } 137 | 138 | # Apply color formatting after padding calculation but before concatenation 139 | if (!is.null(colorize) && is.function(colorize)) { 140 | text <- colorize(text) 141 | } 142 | 143 | paste0(left_pad, text, right_pad) 144 | }, 145 | 146 | #' @description Format the header of the report 147 | fmt_h = function() { 148 | paste0( 149 | " ", 150 | " |", 151 | self$format_column("K", "k", cli::col_green), 152 | "|", 153 | self$format_column("S", "s", cli::col_red), 154 | "|", 155 | self$format_column("E", "e", cli::col_yellow), 156 | "|", 157 | self$format_column("T", "t"), 158 | "|", 159 | self$format_column(" %", "score"), 160 | "|", 161 | self$format_column("Mutator", "mutator"), 162 | "|", 163 | self$format_column("File", "file") 164 | ) 165 | }, 166 | 167 | #' @description Format a row of the report 168 | #' @param status Status symbol (e.g., tick or cross) 169 | #' @param k Number of killed mutations 170 | #' @param s Number of survived mutations 171 | #' @param e Number of errors 172 | #' @param t Total number of mutations 173 | #' @param score Score percentage 174 | #' @param mutator The mutator used 175 | #' @param file The file being tested 176 | #' @return Formatted row string 177 | fmt_r = function(status, k, s, e, t, score, mutator, file) { 178 | paste0( 179 | status, 180 | " |", 181 | self$format_column(as.character(k), "k"), 182 | "|", 183 | self$format_column(as.character(s), "s"), 184 | "|", 185 | self$format_column(as.character(e), "e"), 186 | "|", 187 | self$format_column(as.character(t), "t"), 188 | "|", 189 | self$format_column(score, "score"), 190 | "|", 191 | self$format_column(as.character(mutator), "mutator"), 192 | "|", 193 | self$format_column(file, "file") 194 | ) 195 | }, 196 | 197 | #' @description Initialize a new progress reporter 198 | #' @param test_reporter Reporter to use for testthat::test_dir 199 | #' @param min_time Minimum time to show elapsed time (default: 1s) 200 | #' @param file Output destination (default: stdout) 201 | initialize = function( 202 | test_reporter = "silent", 203 | min_time = 1, 204 | file = stdout() 205 | ) { 206 | super$initialize(test_reporter, file) 207 | 208 | self$min_time <- min_time 209 | self$results <- list() 210 | }, 211 | 212 | #' @description Start reporter 213 | #' @param plan The complete mutation plan 214 | start_reporter = function(plan = NULL) { 215 | super$start_reporter(plan) 216 | self$start_time <- proc.time() 217 | self$results <- list() 218 | self$cat_line(paste(cli::symbol$info, "Mutation Testing")) 219 | self$cat_line(self$fmt_h()) 220 | }, 221 | 222 | #' @description Add a mutation test result 223 | #' @param plan Current testing plan. See `plan()`. 224 | #' @param killed Whether the mutation was killed by tests 225 | #' @param survived Number of survived mutations 226 | #' @param errors Number of errors encountered 227 | #' @md 228 | add_result = function( 229 | plan, 230 | killed, 231 | survived, 232 | errors 233 | ) { 234 | super$add_result(plan, killed, survived, errors) 235 | 236 | status_symbol <- if (killed) { 237 | cli::col_green(cli::symbol$tick) 238 | } else { 239 | cli::col_red("x") 240 | } 241 | 242 | filename <- plan$filename 243 | mutator <- plan$mutator[[1]] 244 | k <- self$results[[filename]]$killed 245 | s <- self$results[[filename]]$survived 246 | t <- self$results[[filename]]$total 247 | e <- self$results[[filename]]$errors 248 | file_name <- basename(filename) 249 | score <- floor(self$current_score * 100) 250 | 251 | # Format and print the row using our formatting function 252 | self$cat_line(self$fmt_r( 253 | status_symbol, 254 | k, 255 | s, 256 | e, 257 | t, 258 | score, 259 | info_oneline(mutator), 260 | file_name 261 | )) 262 | }, 263 | 264 | #' @description Update status spinner (for long-running operations) 265 | #' @param force Force update even if interval hasn't elapsed 266 | update = function(force = FALSE) { 267 | }, 268 | 269 | #' @description End testing current file 270 | end_file = function() { 271 | super$end_file() 272 | }, 273 | 274 | #' @description Carriage return if dynamic, newline otherwise 275 | # nocov start 276 | cr = function() { 277 | }, 278 | # nocov end 279 | 280 | #' @description End reporter with detailed summary 281 | end_reporter = function() { 282 | self$cat_line() 283 | 284 | time <- proc.time() - self$start_time 285 | if (time[[3]] > self$min_time) { 286 | self$cat_line(cli::col_cyan(paste0( 287 | "Duration: ", 288 | sprintf("%.2f s", time[[3]]) 289 | ))) 290 | } 291 | 292 | results <- dplyr::bind_rows(self$results) 293 | k <- sum(results$killed) 294 | s <- sum(results$survived) 295 | t <- sum(results$total) 296 | e <- sum(results$errors) 297 | score <- self$current_score 298 | self$cat_line() 299 | self$rule(cli::style_bold("Results")) 300 | self$cat_line( 301 | "[ ", 302 | cli::col_green("KILLED "), 303 | k, 304 | " | ", 305 | cli::col_red("SURVIVED "), 306 | s, 307 | " | ", 308 | cli::col_yellow("ERRORS "), 309 | e, 310 | " | ", 311 | "TOTAL ", 312 | t, 313 | " | ", 314 | cli::style_bold(cli::col_green(sprintf("SCORE %.1f%%", score * 100))), 315 | " ]" 316 | ) 317 | 318 | self$cat_line() 319 | 320 | super$end_reporter() 321 | } 322 | ) 323 | ) 324 | -------------------------------------------------------------------------------- /R/reporter.R: -------------------------------------------------------------------------------- 1 | #' @title Reporter for Mutation Testing 2 | #' 3 | #' @description 4 | #' The job of a mutation reporter is to aggregate and display the results of mutation tests. 5 | #' It tracks each mutation attempt, reporting on whether the tests killed the mutation or the mutation survived. 6 | #' 7 | #' @field test_reporter Reporter to use for the testthat::test_dir function 8 | #' @field out Output destination for reporter messages 9 | #' @field width Width of the console in characters 10 | #' @field unicode Whether Unicode output is supported 11 | #' @field crayon Whether colored output is supported 12 | #' @field rstudio Whether running in RStudio 13 | #' @field hyperlinks Whether terminal hyperlinks are supported 14 | #' @field current_file Path of the file currently being mutated 15 | #' @field current_mutator Mutator currently being applied 16 | #' @field plan Complete mutation plan for the test run 17 | #' @field results List of mutation test results, indexed by file path 18 | #' @field current_score Current score of the mutation tests 19 | #' 20 | #' @md 21 | #' @export 22 | #' @importFrom rlang `%||%` 23 | #' @family MutationReporter 24 | MutationReporter <- R6::R6Class( 25 | classname = "MutationReporter", 26 | public = list( 27 | # Define test reporter that will be used for running tests 28 | test_reporter = NULL, 29 | 30 | # Output destination 31 | out = NULL, 32 | 33 | # Display settings 34 | width = 80, 35 | unicode = TRUE, 36 | crayon = TRUE, 37 | rstudio = TRUE, 38 | hyperlinks = TRUE, 39 | 40 | # Current state 41 | current_file = NULL, 42 | current_mutator = NULL, 43 | 44 | plan = NULL, 45 | 46 | # Track mutations by file 47 | results = NULL, 48 | current_score = NA_real_, 49 | 50 | #' @description Initialize a new reporter 51 | #' @param test_reporter Reporter to use for the testthat::test_dir function 52 | #' @param file Output destination (default: stdout) 53 | initialize = function(test_reporter = "silent", file = stdout()) { 54 | self$test_reporter <- test_reporter 55 | self$out <- file 56 | 57 | # Capture display settings 58 | self$width <- cli::console_width() 59 | self$unicode <- cli::is_utf8_output() 60 | self$crayon <- cli::num_ansi_colors() > 1 61 | self$rstudio <- Sys.getenv("RSTUDIO") == "1" 62 | self$hyperlinks <- cli::ansi_hyperlink_types()[["run"]] 63 | }, 64 | 65 | #' @description Start reporter 66 | #' @param plan The complete mutation plan 67 | #' @param temp_dir Path to the temporary directory for testing 68 | start_reporter = function(plan = NULL) { 69 | self$plan <- plan 70 | self$results <- list() 71 | self$current_score <- NA_real_ 72 | }, 73 | 74 | #' @description Start testing a file 75 | #' @param filename Path to the file being mutated 76 | start_file = function(filename) { 77 | self$current_file <- filename 78 | self$results[[filename]] <- self$results[[filename]] %||% 79 | list( 80 | total = 0, 81 | killed = 0, 82 | survived = 0, 83 | errors = 0 84 | ) 85 | }, 86 | 87 | #' @description Start testing with a specific mutator 88 | #' @param mutator The mutator being applied 89 | start_mutator = function(mutator) { 90 | self$current_mutator <- mutator 91 | }, 92 | 93 | #' @description Add a mutation test result 94 | #' @param plan Current testing plan. See `plan()`. 95 | #' @param killed Whether the mutation was killed by tests 96 | #' @param survived Number of survived mutations 97 | #' @param errors Number of errors encountered 98 | #' @md 99 | add_result = function( 100 | plan, 101 | killed, 102 | survived, 103 | errors 104 | ) { 105 | filename <- plan$filename 106 | self$results[[filename]]$total <- self$results[[filename]]$total + 1 107 | self$results[[filename]]$killed <- self$results[[filename]]$killed + 108 | killed 109 | self$results[[filename]]$survived <- self$results[[filename]]$survived + 110 | survived 111 | self$results[[filename]]$errors <- self$results[[filename]]$errors + 112 | errors 113 | killed_counts <- purrr::map(self$results, "killed") 114 | total_counts <- purrr::map(self$results, "total") 115 | self$current_score <- sum(as.numeric(killed_counts)) / 116 | sum(as.numeric(total_counts)) 117 | }, 118 | 119 | #' @description End testing with current mutator 120 | end_mutator = function() { 121 | self$current_mutator <- NULL 122 | }, 123 | 124 | #' @description End testing current file 125 | end_file = function() { 126 | self$current_file <- NULL 127 | }, 128 | 129 | #' @description End reporter and show summary 130 | end_reporter = function() { 131 | }, 132 | 133 | #' @description Get the current score 134 | get_score = function() { 135 | self$current_score 136 | }, 137 | 138 | #' @description Print a message to the output 139 | #' @param ... Message to print 140 | # nocov start 141 | cat_tight = function(...) { 142 | cat(..., sep = "", file = self$out, append = TRUE) 143 | }, 144 | # nocov end 145 | 146 | #' @description Print a message to the output 147 | #' @param ... Message to print 148 | cat_line = function(...) { 149 | cli::cat_line(..., file = self$out) 150 | }, 151 | 152 | #' @description Print a message to the output with a rule 153 | #' @param ... Message to print 154 | rule = function(...) { 155 | cli::cat_rule(..., file = self$out) 156 | } 157 | ) 158 | ) 159 | 160 | #' Create a default reporter 161 | #' 162 | #' @param ... Arguments passed to the `?ProgressMutationReporter` constructor. 163 | #' @md 164 | #' @export 165 | #' @family MutationReporter 166 | default_reporter <- function(...) { 167 | ProgressMutationReporter$new(...) 168 | } 169 | -------------------------------------------------------------------------------- /R/test_strategy.R: -------------------------------------------------------------------------------- 1 | #' @import testthat 2 | NULL 3 | 4 | #' @title TestStrategy interface 5 | #' 6 | #' @description 7 | #' Extend this class to implement a custom test strategy. 8 | #' 9 | #' @export 10 | #' @md 11 | #' @family TestStrategy 12 | TestStrategy <- R6::R6Class( 13 | classname = "TestStrategy", 14 | public = list( 15 | #' @description Execute the test strategy 16 | #' @param path The path to the test directory 17 | #' @param plan The current mutation plan. See `plan()`. 18 | #' @param reporter The reporter to use for test results 19 | #' @return The test result 20 | execute = function(path, plan, reporter) { 21 | stop("Not implemented") 22 | } 23 | ) 24 | ) 25 | 26 | #' @title Run all tests for a mutant 27 | #' 28 | #' @description 29 | #' This test strategy tells if a mutant is caught by any test. 30 | #' 31 | #' To get faster results, especially for big codebases, use `?FileTestStrategy` instead. 32 | #' 33 | #' @export 34 | #' @md 35 | #' @family TestStrategy 36 | FullTestStrategy <- R6::R6Class( 37 | classname = "FullTestStrategy", 38 | inherit = TestStrategy, 39 | private = list( 40 | args = list() 41 | ), 42 | public = list( 43 | #' @description Initialize 44 | #' @param load_helpers Whether to load test helpers 45 | #' @param load_package The package loading strategy 46 | initialize = function( 47 | load_helpers = TRUE, 48 | load_package = c("source", "none", "installed") 49 | ) { 50 | private$args <- list( 51 | load_helpers = load_helpers, 52 | load_package = load_package 53 | ) 54 | }, 55 | #' @description Execute the test strategy 56 | #' @param path The path to the test directory 57 | #' @param plan The current mutation plan. See `plan()`. 58 | #' @param reporter The reporter to use for test results 59 | #' @return The test results 60 | execute = function(path, plan, reporter) { 61 | testthat::test_dir( 62 | path, 63 | stop_on_failure = FALSE, 64 | reporter = reporter, 65 | load_helpers = private$args$load_helpers, 66 | load_package = private$args$load_package 67 | ) 68 | } 69 | ) 70 | ) 71 | 72 | #' @title Run tests matching the mutated source file name 73 | #' 74 | #' @description 75 | #' This strategy tells if a mutant is caught by a test matching the source file name. 76 | #' 77 | #' For example, if the source file name is `foo.R`, and there are test files named `test-foo.R` or `test-bar.R`, 78 | #' only `test-foo.R` will be run. 79 | #' 80 | #' This strategy should give faster results than `?FullTestStrategy`, especially for big codebases, 81 | #' but the score might be less accurate. 82 | #' 83 | #' @export 84 | #' @md 85 | #' @family TestStrategy 86 | FileTestStrategy <- R6::R6Class( 87 | classname = "FileTestStrategy", 88 | inherit = TestStrategy, 89 | private = list( 90 | args = list() 91 | ), 92 | public = list( 93 | #' @description Initialize the FileTestStrategy 94 | #' @param load_helpers Whether to load test helpers 95 | #' @param load_package The package loading strategy 96 | initialize = function( 97 | load_helpers = TRUE, 98 | load_package = c("source", "none", "installed") 99 | ) { 100 | private$args <- list( 101 | load_helpers = load_helpers, 102 | load_package = load_package 103 | ) 104 | }, 105 | #' @description Execute the test strategy 106 | #' @param path The path to the test directory 107 | #' @param plan The current mutation plan. See `plan()`. 108 | #' @param reporter The reporter to use for test results 109 | #' @return The test results 110 | execute = function(path, plan, reporter) { 111 | file_name <- tools::file_path_sans_ext(basename(plan$filename)) 112 | if (!any(grepl(file_name, list.files(path)))) { 113 | return(.empty_test_result()) 114 | } 115 | testthat::test_dir( 116 | path = path, 117 | filter = file_name, 118 | stop_on_failure = FALSE, 119 | reporter = reporter, 120 | load_helpers = private$args$load_helpers, 121 | load_package = private$args$load_package 122 | ) 123 | } 124 | ) 125 | ) 126 | 127 | #' @title Create a default run strategy 128 | #' 129 | #' @param ... Arguments passed to the `?FullTestStrategy` constructor. 130 | #' @return A `?TestStrategy` object 131 | #' 132 | #' @export 133 | #' @md 134 | #' @family TestStrategy 135 | default_test_strategy <- function(...) { 136 | FullTestStrategy$new(...) 137 | } 138 | 139 | .empty_test_result <- function() { 140 | structure( 141 | list(), 142 | class = c("testthat_results") 143 | ) 144 | } 145 | -------------------------------------------------------------------------------- /README.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | output: github_document 3 | --- 4 | 5 | 6 | 7 | ```{r, include = FALSE} 8 | knitr::opts_chunk$set( 9 | collapse = TRUE, 10 | comment = "#>", 11 | fig.path = "man/figures/README-", 12 | out.width = "100%", 13 | root.dir = system.file("examples", "operators", package = "muttest") 14 | ) 15 | 16 | include_file <- function(file, filename) { 17 | code <- readLines(file) 18 | cat("```r", paste("#'", filename), code, "```", sep = "\n") 19 | } 20 | ``` 21 | # muttest 22 | 23 | 24 | [![CRAN status](https://www.r-pkg.org/badges/version/muttest)](https://CRAN.R-project.org/package=muttest) 25 | [![R-CMD-check](https://github.com/jakubsob/muttest/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/jakubsob/muttest/actions/workflows/R-CMD-check.yaml) 26 | [![Codecov test coverage](https://codecov.io/gh/jakubsob/muttest/graph/badge.svg)](https://app.codecov.io/gh/jakubsob/muttest) 27 | [![cucumber](https://img.shields.io/github/actions/workflow/status/jakubsob/muttest/test-acceptance.yaml?branch=main&label=cucumber&logo=cucumber&color=23D96C&labelColor=0f2a13)](https://github.com/jakubsob/muttest/actions/workflows/test-acceptance.yaml) 28 | [![muttest](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/jakubsob/muttest/badges/.badges/muttest.json)](https://github.com/jakubsob/muttest/actions/workflows/test-mutation.yaml) 29 | [![Grand total](http://cranlogs.r-pkg.org/badges/grand-total/muttest)](https://cran.r-project.org/package=muttest) 30 | [![Last month](http://cranlogs.r-pkg.org/badges/last-month/muttest)](https://cran.r-project.org/package=muttest) 31 | 32 | 33 | Measure quality of your tests with **{muttest}**. 34 | 35 | [covr](https://github.com/r-lib/covr) tells you how much of your code is executed by tests, but it tells you nothing about the quality of those tests. 36 | 37 | In fact, you can have tests with zero assertions and still get 100% coverage. That can give a false sense of security. Mutation testing addresses this gap. 38 | 39 | It works like this: 40 | 41 | - Define a set of code changes (mutations). 42 | - Run your test suite against mutated versions of your source code. 43 | - Measure how often the mutations are caught (i.e., cause test failures). 44 | 45 | This reveals whether your tests are asserting the right things: 46 | 47 | - 0% score → Your tests pass no matter what changes. Your assertions are weak. 48 | - 100% score → Every mutation triggers a test failure. Your tests are robust. 49 | 50 | {muttest} not only gives you the score, but it also tells you tests for which files require improved assertions. 51 | 52 | # Example 53 | 54 | Given our codebase is: 55 | 56 | ```{r} 57 | #| echo: false 58 | #| results: asis 59 | include_file( 60 | system.file("examples", "operators", "R", "calculate.R", package = "muttest"), 61 | "R/calculate.R" 62 | ) 63 | ``` 64 | 65 | And our tests are: 66 | 67 | ```{r} 68 | #| echo: false 69 | #| results: asis 70 | include_file( 71 | system.file("examples", "operators", "tests", "testthat", "test-calculate.R", package = "muttest"), 72 | "tests/testthat/test_calculate.R" 73 | ) 74 | ``` 75 | 76 | 77 | When running `muttest::muttest()` we'll get a report of the mutation score: 78 | 79 | ```{r} 80 | #| eval: false 81 | plan <- muttest::plan( 82 | source_files = "R/calculate.R", 83 | mutators = list( 84 | muttest::operator("+", "-"), 85 | muttest::operator("*", "/") 86 | ) 87 | ) 88 | 89 | muttest::muttest(plan) 90 | #> ℹ Mutation Testing 91 | #> | K | S | E | T | % | Mutator | File 92 | #> x | 0 | 1 | 0 | 1 | 0 | + → - | calculate.R 93 | #> ✔ | 1 | 1 | 0 | 2 | 50 | * → / | calculate.R 94 | #> ── Mutation Testing Results ──────────────────────────────────────────────────── 95 | #> [ KILLED 1 | SURVIVED 1 | ERRORS 0 | TOTAL 2 | SCORE 50.0% ] 96 | ``` 97 | 98 | The mutation score is: $\text{Mutation Score} = \frac{\text{Killed Mutants}}{\text{Total Mutants}} \times 100\%$, where a Mutant is defined as variant of the original code that is used to test the robustness of the test suite. 99 | 100 | In the example there were 2 mutants of the code: 101 | 102 | ```r 103 | #' R/calculate.R 104 | calculate <- function(x, y) { 105 | (x - y) * 0 # mutant 1: "+" -> "-" 106 | } 107 | ``` 108 | 109 | ```r 110 | #' R/calculate.R 111 | calculate <- function(x, y) { 112 | (x + y) / 0 # mutant 2: "*" -> "/" 113 | } 114 | ``` 115 | 116 | Tests are run against both variants of the code. 117 | 118 | The first test run against the first mutant will pass, because the result is still 0. The second test run against the second mutant will fail, because the result is Inf. 119 | 120 | The second test will pass against both mutants, because the result is still numeric. 121 | 122 | ```r 123 | #' tests/testthat/test_calculate.R 124 | test_that("calculate always returns 0", { 125 | # 🟢 This test doesn't kill "+" -> "-" operator mutant: (2 - 2) * 0 = 0 126 | # ❌ This test kills "*" -> "/" operator mutant: (2 + 2) / 0 = Inf 127 | expect_equal(calculate(2, 2), 0) 128 | }) 129 | 130 | test_that("calculate returns a numeric", { 131 | # 🟢 This test doesn't kill "+" -> "-", (2 - 2) * 0 = 0, is numeric 132 | # 🟢 This test doesn't kill "*" -> "/", (2 + 2) / 0 = Inf, is numeric 133 | expect_true(is.numeric(calculate(2, 2))) 134 | }) 135 | ``` 136 | 137 | We have killed 1 mutant out of 2, so the mutation score is 50%. 138 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # muttest 5 | 6 | 7 | 8 | [![CRAN 9 | status](https://www.r-pkg.org/badges/version/muttest)](https://CRAN.R-project.org/package=muttest) 10 | [![R-CMD-check](https://github.com/jakubsob/muttest/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/jakubsob/muttest/actions/workflows/R-CMD-check.yaml) 11 | [![Codecov test 12 | coverage](https://codecov.io/gh/jakubsob/muttest/graph/badge.svg)](https://app.codecov.io/gh/jakubsob/muttest) 13 | [![cucumber](https://img.shields.io/github/actions/workflow/status/jakubsob/muttest/test-acceptance.yaml?branch=main&label=cucumber&logo=cucumber&color=23D96C&labelColor=0f2a13)](https://github.com/jakubsob/muttest/actions/workflows/test-acceptance.yaml) 14 | [![muttest](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/jakubsob/muttest/badges/.badges/muttest.json)](https://github.com/jakubsob/muttest/actions/workflows/test-mutation.yaml) 15 | [![Grand 16 | total](http://cranlogs.r-pkg.org/badges/grand-total/muttest)](https://cran.r-project.org/package=muttest) 17 | [![Last 18 | month](http://cranlogs.r-pkg.org/badges/last-month/muttest)](https://cran.r-project.org/package=muttest) 19 | 20 | 21 | Measure quality of your tests with **{muttest}**. 22 | 23 | [covr](https://github.com/r-lib/covr) tells you how much of your code is 24 | executed by tests, but it tells you nothing about the quality of those 25 | tests. 26 | 27 | In fact, you can have tests with zero assertions and still get 100% 28 | coverage. That can give a false sense of security. Mutation testing 29 | addresses this gap. 30 | 31 | It works like this: 32 | 33 | - Define a set of code changes (mutations). 34 | - Run your test suite against mutated versions of your source code. 35 | - Measure how often the mutations are caught (i.e., cause test 36 | failures). 37 | 38 | This reveals whether your tests are asserting the right things: 39 | 40 | - 0% score → Your tests pass no matter what changes. Your assertions are 41 | weak. 42 | - 100% score → Every mutation triggers a test failure. Your tests are 43 | robust. 44 | 45 | {muttest} not only gives you the score, but it also tells you tests for 46 | which files require improved assertions. 47 | 48 | # Example 49 | 50 | Given our codebase is: 51 | 52 | ``` r 53 | #' R/calculate.R 54 | calculate <- function(x, y) { 55 | (x + y) * 0 56 | } 57 | ``` 58 | 59 | And our tests are: 60 | 61 | ``` r 62 | #' tests/testthat/test_calculate.R 63 | test_that("calculate returns a numeric", { 64 | expect_true(is.numeric(calculate(2, 2))) # ❌ This assertion doesn't kill mutants 65 | }) 66 | 67 | test_that("calculate always returns 0", { 68 | expect_equal(calculate(2, 2), 0) # ✅ This assertion only kills "*" -> "/" mutant 69 | }) 70 | ``` 71 | 72 | When running `muttest::muttest()` we’ll get a report of the mutation 73 | score: 74 | 75 | ``` r 76 | plan <- muttest::plan( 77 | source_files = "R/calculate.R", 78 | mutators = list( 79 | muttest::operator("+", "-"), 80 | muttest::operator("*", "/") 81 | ) 82 | ) 83 | 84 | muttest::muttest(plan) 85 | #> ℹ Mutation Testing 86 | #> | K | S | E | T | % | Mutator | File 87 | #> x | 0 | 1 | 0 | 1 | 0 | + → - | calculate.R 88 | #> ✔ | 1 | 1 | 0 | 2 | 50 | * → / | calculate.R 89 | #> ── Mutation Testing Results ──────────────────────────────────────────────────── 90 | #> [ KILLED 1 | SURVIVED 1 | ERRORS 0 | TOTAL 2 | SCORE 50.0% ] 91 | ``` 92 | 93 | The mutation score is: 94 | $\text{Mutation Score} = \frac{\text{Killed Mutants}}{\text{Total Mutants}} \times 100\%$, 95 | where a Mutant is defined as variant of the original code that is used 96 | to test the robustness of the test suite. 97 | 98 | In the example there were 2 mutants of the code: 99 | 100 | ``` r 101 | #' R/calculate.R 102 | calculate <- function(x, y) { 103 | (x - y) * 0 # mutant 1: "+" -> "-" 104 | } 105 | ``` 106 | 107 | ``` r 108 | #' R/calculate.R 109 | calculate <- function(x, y) { 110 | (x + y) / 0 # mutant 2: "*" -> "/" 111 | } 112 | ``` 113 | 114 | Tests are run against both variants of the code. 115 | 116 | The first test run against the first mutant will pass, because the 117 | result is still 0. The second test run against the second mutant will 118 | fail, because the result is Inf. 119 | 120 | The second test will pass against both mutants, because the result is 121 | still numeric. 122 | 123 | ``` r 124 | #' tests/testthat/test_calculate.R 125 | test_that("calculate always returns 0", { 126 | # 🟢 This test doesn't kill "+" -> "-" operator mutant: (2 - 2) * 0 = 0 127 | # ❌ This test kills "*" -> "/" operator mutant: (2 + 2) / 0 = Inf 128 | expect_equal(calculate(2, 2), 0) 129 | }) 130 | 131 | test_that("calculate returns a numeric", { 132 | # 🟢 This test doesn't kill "+" -> "-", (2 - 2) * 0 = 0, is numeric 133 | # 🟢 This test doesn't kill "*" -> "/", (2 + 2) / 0 = Inf, is numeric 134 | expect_true(is.numeric(calculate(2, 2))) 135 | }) 136 | ``` 137 | 138 | We have killed 1 mutant out of 2, so the mutation score is 50%. 139 | -------------------------------------------------------------------------------- /_pkgdown.yml: -------------------------------------------------------------------------------- 1 | url: https://jakubsob.github.io/muttest/ 2 | template: 3 | bootstrap: 5 4 | light-switch: true 5 | bslib: 6 | bg: '#ffffff' 7 | fg: '#000000' 8 | primary: '#886868' 9 | secondary: '#F77669' 10 | includes: 11 | in_header: | 12 | 13 | 14 | 21 | 22 | navbar: 23 | type: dark 24 | structure: 25 | left: [reference, articles, news] 26 | components: 27 | articles: 28 | text: Articles 29 | menu: 30 | - text: How it works 31 | href: articles/how-it-works.html 32 | 33 | reference: 34 | - title: Run 35 | contents: 36 | - muttest 37 | - plan 38 | - title: Mutators 39 | contents: 40 | - operator 41 | - title: Test run strategy 42 | contents: 43 | - default_test_strategy 44 | - FullTestStrategy 45 | - FileTestStrategy 46 | - TestStrategy 47 | - title: Project copy strategy 48 | contents: 49 | - default_copy_strategy 50 | - PackageCopyStrategy 51 | - CopyStrategy 52 | - title: Test reporting 53 | contents: 54 | - default_reporter 55 | - ProgressMutationReporter 56 | - MutationReporter 57 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | comment: false 2 | 3 | coverage: 4 | status: 5 | project: 6 | default: 7 | target: auto 8 | threshold: 1% 9 | informational: true 10 | patch: 11 | default: 12 | target: auto 13 | threshold: 1% 14 | informational: true 15 | -------------------------------------------------------------------------------- /cran-comments.md: -------------------------------------------------------------------------------- 1 | Fixed package name in Description field, muttest -> 'muttest'. 2 | -------------------------------------------------------------------------------- /inst/examples/operators/DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: example 2 | Version: 0.0.0.9000 3 | -------------------------------------------------------------------------------- /inst/examples/operators/R/calculate.R: -------------------------------------------------------------------------------- 1 | calculate <- function(x, y) { 2 | (x + y) * 0 3 | } 4 | -------------------------------------------------------------------------------- /inst/examples/operators/tests/testthat/helper.R: -------------------------------------------------------------------------------- 1 | helper_func <- function() { 2 | } 3 | -------------------------------------------------------------------------------- /inst/examples/operators/tests/testthat/setup.R: -------------------------------------------------------------------------------- 1 | setup_func <- function() { 2 | } 3 | -------------------------------------------------------------------------------- /inst/examples/operators/tests/testthat/test-calculate.R: -------------------------------------------------------------------------------- 1 | test_that("calculate returns a numeric", { 2 | expect_true(is.numeric(calculate(2, 2))) # ❌ This assertion doesn't kill mutants 3 | }) 4 | 5 | test_that("calculate always returns 0", { 6 | expect_equal(calculate(2, 2), 0) # ✅ This assertion only kills "*" -> "/" mutant 7 | }) 8 | -------------------------------------------------------------------------------- /man/CopyStrategy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/project_copy_strategy.R 3 | \name{CopyStrategy} 4 | \alias{CopyStrategy} 5 | \title{CopyStrategy interface} 6 | \description{ 7 | Extend this class to implement a custom copy strategy. 8 | } 9 | \seealso{ 10 | Other CopyStrategy: 11 | \code{\link{PackageCopyStrategy}}, 12 | \code{\link{default_copy_strategy}()} 13 | } 14 | \concept{CopyStrategy} 15 | \section{Methods}{ 16 | \subsection{Public methods}{ 17 | \itemize{ 18 | \item \href{#method-CopyStrategy-execute}{\code{CopyStrategy$execute()}} 19 | \item \href{#method-CopyStrategy-clone}{\code{CopyStrategy$clone()}} 20 | } 21 | } 22 | \if{html}{\out{
}} 23 | \if{html}{\out{}} 24 | \if{latex}{\out{\hypertarget{method-CopyStrategy-execute}{}}} 25 | \subsection{Method \code{execute()}}{ 26 | Copy project files according to the strategy 27 | \subsection{Usage}{ 28 | \if{html}{\out{
}}\preformatted{CopyStrategy$execute(original_dir)}\if{html}{\out{
}} 29 | } 30 | 31 | \subsection{Arguments}{ 32 | \if{html}{\out{
}} 33 | \describe{ 34 | \item{\code{original_dir}}{The original directory to copy from} 35 | 36 | \item{\code{plan}}{The current test plan} 37 | } 38 | \if{html}{\out{
}} 39 | } 40 | \subsection{Returns}{ 41 | The path to the temporary directory 42 | } 43 | } 44 | \if{html}{\out{
}} 45 | \if{html}{\out{}} 46 | \if{latex}{\out{\hypertarget{method-CopyStrategy-clone}{}}} 47 | \subsection{Method \code{clone()}}{ 48 | The objects of this class are cloneable with this method. 49 | \subsection{Usage}{ 50 | \if{html}{\out{
}}\preformatted{CopyStrategy$clone(deep = FALSE)}\if{html}{\out{
}} 51 | } 52 | 53 | \subsection{Arguments}{ 54 | \if{html}{\out{
}} 55 | \describe{ 56 | \item{\code{deep}}{Whether to make a deep clone.} 57 | } 58 | \if{html}{\out{
}} 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /man/FileTestStrategy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/test_strategy.R 3 | \name{FileTestStrategy} 4 | \alias{FileTestStrategy} 5 | \title{Run tests matching the mutated source file name} 6 | \description{ 7 | This strategy tells if a mutant is caught by a test matching the source file name. 8 | 9 | For example, if the source file name is \code{foo.R}, and there are test files named \code{test-foo.R} or \code{test-bar.R}, 10 | only \code{test-foo.R} will be run. 11 | 12 | This strategy should give faster results than \code{?FullTestStrategy}, especially for big codebases, 13 | but the score might be less accurate. 14 | } 15 | \seealso{ 16 | Other TestStrategy: 17 | \code{\link{FullTestStrategy}}, 18 | \code{\link{TestStrategy}}, 19 | \code{\link{default_test_strategy}()} 20 | } 21 | \concept{TestStrategy} 22 | \section{Super class}{ 23 | \code{\link[muttest:TestStrategy]{muttest::TestStrategy}} -> \code{FileTestStrategy} 24 | } 25 | \section{Methods}{ 26 | \subsection{Public methods}{ 27 | \itemize{ 28 | \item \href{#method-FileTestStrategy-new}{\code{FileTestStrategy$new()}} 29 | \item \href{#method-FileTestStrategy-execute}{\code{FileTestStrategy$execute()}} 30 | \item \href{#method-FileTestStrategy-clone}{\code{FileTestStrategy$clone()}} 31 | } 32 | } 33 | \if{html}{\out{
}} 34 | \if{html}{\out{}} 35 | \if{latex}{\out{\hypertarget{method-FileTestStrategy-new}{}}} 36 | \subsection{Method \code{new()}}{ 37 | Initialize the FileTestStrategy 38 | \subsection{Usage}{ 39 | \if{html}{\out{
}}\preformatted{FileTestStrategy$new( 40 | load_helpers = TRUE, 41 | load_package = c("source", "none", "installed") 42 | )}\if{html}{\out{
}} 43 | } 44 | 45 | \subsection{Arguments}{ 46 | \if{html}{\out{
}} 47 | \describe{ 48 | \item{\code{load_helpers}}{Whether to load test helpers} 49 | 50 | \item{\code{load_package}}{The package loading strategy} 51 | } 52 | \if{html}{\out{
}} 53 | } 54 | } 55 | \if{html}{\out{
}} 56 | \if{html}{\out{}} 57 | \if{latex}{\out{\hypertarget{method-FileTestStrategy-execute}{}}} 58 | \subsection{Method \code{execute()}}{ 59 | Execute the test strategy 60 | \subsection{Usage}{ 61 | \if{html}{\out{
}}\preformatted{FileTestStrategy$execute(path, plan, reporter)}\if{html}{\out{
}} 62 | } 63 | 64 | \subsection{Arguments}{ 65 | \if{html}{\out{
}} 66 | \describe{ 67 | \item{\code{path}}{The path to the test directory} 68 | 69 | \item{\code{plan}}{The current mutation plan. See \code{plan()}.} 70 | 71 | \item{\code{reporter}}{The reporter to use for test results} 72 | } 73 | \if{html}{\out{
}} 74 | } 75 | \subsection{Returns}{ 76 | The test results 77 | } 78 | } 79 | \if{html}{\out{
}} 80 | \if{html}{\out{}} 81 | \if{latex}{\out{\hypertarget{method-FileTestStrategy-clone}{}}} 82 | \subsection{Method \code{clone()}}{ 83 | The objects of this class are cloneable with this method. 84 | \subsection{Usage}{ 85 | \if{html}{\out{
}}\preformatted{FileTestStrategy$clone(deep = FALSE)}\if{html}{\out{
}} 86 | } 87 | 88 | \subsection{Arguments}{ 89 | \if{html}{\out{
}} 90 | \describe{ 91 | \item{\code{deep}}{Whether to make a deep clone.} 92 | } 93 | \if{html}{\out{
}} 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /man/FullTestStrategy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/test_strategy.R 3 | \name{FullTestStrategy} 4 | \alias{FullTestStrategy} 5 | \title{Run all tests for a mutant} 6 | \description{ 7 | This test strategy tells if a mutant is caught by any test. 8 | 9 | To get faster results, especially for big codebases, use \code{?FileTestStrategy} instead. 10 | } 11 | \seealso{ 12 | Other TestStrategy: 13 | \code{\link{FileTestStrategy}}, 14 | \code{\link{TestStrategy}}, 15 | \code{\link{default_test_strategy}()} 16 | } 17 | \concept{TestStrategy} 18 | \section{Super class}{ 19 | \code{\link[muttest:TestStrategy]{muttest::TestStrategy}} -> \code{FullTestStrategy} 20 | } 21 | \section{Methods}{ 22 | \subsection{Public methods}{ 23 | \itemize{ 24 | \item \href{#method-FullTestStrategy-new}{\code{FullTestStrategy$new()}} 25 | \item \href{#method-FullTestStrategy-execute}{\code{FullTestStrategy$execute()}} 26 | \item \href{#method-FullTestStrategy-clone}{\code{FullTestStrategy$clone()}} 27 | } 28 | } 29 | \if{html}{\out{
}} 30 | \if{html}{\out{}} 31 | \if{latex}{\out{\hypertarget{method-FullTestStrategy-new}{}}} 32 | \subsection{Method \code{new()}}{ 33 | Initialize 34 | \subsection{Usage}{ 35 | \if{html}{\out{
}}\preformatted{FullTestStrategy$new( 36 | load_helpers = TRUE, 37 | load_package = c("source", "none", "installed") 38 | )}\if{html}{\out{
}} 39 | } 40 | 41 | \subsection{Arguments}{ 42 | \if{html}{\out{
}} 43 | \describe{ 44 | \item{\code{load_helpers}}{Whether to load test helpers} 45 | 46 | \item{\code{load_package}}{The package loading strategy} 47 | } 48 | \if{html}{\out{
}} 49 | } 50 | } 51 | \if{html}{\out{
}} 52 | \if{html}{\out{}} 53 | \if{latex}{\out{\hypertarget{method-FullTestStrategy-execute}{}}} 54 | \subsection{Method \code{execute()}}{ 55 | Execute the test strategy 56 | \subsection{Usage}{ 57 | \if{html}{\out{
}}\preformatted{FullTestStrategy$execute(path, plan, reporter)}\if{html}{\out{
}} 58 | } 59 | 60 | \subsection{Arguments}{ 61 | \if{html}{\out{
}} 62 | \describe{ 63 | \item{\code{path}}{The path to the test directory} 64 | 65 | \item{\code{plan}}{The current mutation plan. See \code{plan()}.} 66 | 67 | \item{\code{reporter}}{The reporter to use for test results} 68 | } 69 | \if{html}{\out{
}} 70 | } 71 | \subsection{Returns}{ 72 | The test results 73 | } 74 | } 75 | \if{html}{\out{
}} 76 | \if{html}{\out{}} 77 | \if{latex}{\out{\hypertarget{method-FullTestStrategy-clone}{}}} 78 | \subsection{Method \code{clone()}}{ 79 | The objects of this class are cloneable with this method. 80 | \subsection{Usage}{ 81 | \if{html}{\out{
}}\preformatted{FullTestStrategy$clone(deep = FALSE)}\if{html}{\out{
}} 82 | } 83 | 84 | \subsection{Arguments}{ 85 | \if{html}{\out{
}} 86 | \describe{ 87 | \item{\code{deep}}{Whether to make a deep clone.} 88 | } 89 | \if{html}{\out{
}} 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /man/MutationReporter.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/reporter.R 3 | \name{MutationReporter} 4 | \alias{MutationReporter} 5 | \title{Reporter for Mutation Testing} 6 | \description{ 7 | The job of a mutation reporter is to aggregate and display the results of mutation tests. 8 | It tracks each mutation attempt, reporting on whether the tests killed the mutation or the mutation survived. 9 | } 10 | \seealso{ 11 | Other MutationReporter: 12 | \code{\link{ProgressMutationReporter}}, 13 | \code{\link{default_reporter}()} 14 | } 15 | \concept{MutationReporter} 16 | \section{Public fields}{ 17 | \if{html}{\out{
}} 18 | \describe{ 19 | \item{\code{test_reporter}}{Reporter to use for the testthat::test_dir function} 20 | 21 | \item{\code{out}}{Output destination for reporter messages} 22 | 23 | \item{\code{width}}{Width of the console in characters} 24 | 25 | \item{\code{unicode}}{Whether Unicode output is supported} 26 | 27 | \item{\code{crayon}}{Whether colored output is supported} 28 | 29 | \item{\code{rstudio}}{Whether running in RStudio} 30 | 31 | \item{\code{hyperlinks}}{Whether terminal hyperlinks are supported} 32 | 33 | \item{\code{current_file}}{Path of the file currently being mutated} 34 | 35 | \item{\code{current_mutator}}{Mutator currently being applied} 36 | 37 | \item{\code{plan}}{Complete mutation plan for the test run} 38 | 39 | \item{\code{results}}{List of mutation test results, indexed by file path} 40 | 41 | \item{\code{current_score}}{Current score of the mutation tests} 42 | } 43 | \if{html}{\out{
}} 44 | } 45 | \section{Methods}{ 46 | \subsection{Public methods}{ 47 | \itemize{ 48 | \item \href{#method-MutationReporter-new}{\code{MutationReporter$new()}} 49 | \item \href{#method-MutationReporter-start_reporter}{\code{MutationReporter$start_reporter()}} 50 | \item \href{#method-MutationReporter-start_file}{\code{MutationReporter$start_file()}} 51 | \item \href{#method-MutationReporter-start_mutator}{\code{MutationReporter$start_mutator()}} 52 | \item \href{#method-MutationReporter-add_result}{\code{MutationReporter$add_result()}} 53 | \item \href{#method-MutationReporter-end_mutator}{\code{MutationReporter$end_mutator()}} 54 | \item \href{#method-MutationReporter-end_file}{\code{MutationReporter$end_file()}} 55 | \item \href{#method-MutationReporter-end_reporter}{\code{MutationReporter$end_reporter()}} 56 | \item \href{#method-MutationReporter-get_score}{\code{MutationReporter$get_score()}} 57 | \item \href{#method-MutationReporter-cat_tight}{\code{MutationReporter$cat_tight()}} 58 | \item \href{#method-MutationReporter-cat_line}{\code{MutationReporter$cat_line()}} 59 | \item \href{#method-MutationReporter-rule}{\code{MutationReporter$rule()}} 60 | \item \href{#method-MutationReporter-clone}{\code{MutationReporter$clone()}} 61 | } 62 | } 63 | \if{html}{\out{
}} 64 | \if{html}{\out{}} 65 | \if{latex}{\out{\hypertarget{method-MutationReporter-new}{}}} 66 | \subsection{Method \code{new()}}{ 67 | Initialize a new reporter 68 | \subsection{Usage}{ 69 | \if{html}{\out{
}}\preformatted{MutationReporter$new(test_reporter = "silent", file = stdout())}\if{html}{\out{
}} 70 | } 71 | 72 | \subsection{Arguments}{ 73 | \if{html}{\out{
}} 74 | \describe{ 75 | \item{\code{test_reporter}}{Reporter to use for the testthat::test_dir function} 76 | 77 | \item{\code{file}}{Output destination (default: stdout)} 78 | } 79 | \if{html}{\out{
}} 80 | } 81 | } 82 | \if{html}{\out{
}} 83 | \if{html}{\out{}} 84 | \if{latex}{\out{\hypertarget{method-MutationReporter-start_reporter}{}}} 85 | \subsection{Method \code{start_reporter()}}{ 86 | Start reporter 87 | \subsection{Usage}{ 88 | \if{html}{\out{
}}\preformatted{MutationReporter$start_reporter(plan = NULL)}\if{html}{\out{
}} 89 | } 90 | 91 | \subsection{Arguments}{ 92 | \if{html}{\out{
}} 93 | \describe{ 94 | \item{\code{plan}}{The complete mutation plan} 95 | 96 | \item{\code{temp_dir}}{Path to the temporary directory for testing} 97 | } 98 | \if{html}{\out{
}} 99 | } 100 | } 101 | \if{html}{\out{
}} 102 | \if{html}{\out{}} 103 | \if{latex}{\out{\hypertarget{method-MutationReporter-start_file}{}}} 104 | \subsection{Method \code{start_file()}}{ 105 | Start testing a file 106 | \subsection{Usage}{ 107 | \if{html}{\out{
}}\preformatted{MutationReporter$start_file(filename)}\if{html}{\out{
}} 108 | } 109 | 110 | \subsection{Arguments}{ 111 | \if{html}{\out{
}} 112 | \describe{ 113 | \item{\code{filename}}{Path to the file being mutated} 114 | } 115 | \if{html}{\out{
}} 116 | } 117 | } 118 | \if{html}{\out{
}} 119 | \if{html}{\out{}} 120 | \if{latex}{\out{\hypertarget{method-MutationReporter-start_mutator}{}}} 121 | \subsection{Method \code{start_mutator()}}{ 122 | Start testing with a specific mutator 123 | \subsection{Usage}{ 124 | \if{html}{\out{
}}\preformatted{MutationReporter$start_mutator(mutator)}\if{html}{\out{
}} 125 | } 126 | 127 | \subsection{Arguments}{ 128 | \if{html}{\out{
}} 129 | \describe{ 130 | \item{\code{mutator}}{The mutator being applied} 131 | } 132 | \if{html}{\out{
}} 133 | } 134 | } 135 | \if{html}{\out{
}} 136 | \if{html}{\out{}} 137 | \if{latex}{\out{\hypertarget{method-MutationReporter-add_result}{}}} 138 | \subsection{Method \code{add_result()}}{ 139 | Add a mutation test result 140 | \subsection{Usage}{ 141 | \if{html}{\out{
}}\preformatted{MutationReporter$add_result(plan, killed, survived, errors)}\if{html}{\out{
}} 142 | } 143 | 144 | \subsection{Arguments}{ 145 | \if{html}{\out{
}} 146 | \describe{ 147 | \item{\code{plan}}{Current testing plan. See \code{plan()}.} 148 | 149 | \item{\code{killed}}{Whether the mutation was killed by tests} 150 | 151 | \item{\code{survived}}{Number of survived mutations} 152 | 153 | \item{\code{errors}}{Number of errors encountered} 154 | } 155 | \if{html}{\out{
}} 156 | } 157 | } 158 | \if{html}{\out{
}} 159 | \if{html}{\out{}} 160 | \if{latex}{\out{\hypertarget{method-MutationReporter-end_mutator}{}}} 161 | \subsection{Method \code{end_mutator()}}{ 162 | End testing with current mutator 163 | \subsection{Usage}{ 164 | \if{html}{\out{
}}\preformatted{MutationReporter$end_mutator()}\if{html}{\out{
}} 165 | } 166 | 167 | } 168 | \if{html}{\out{
}} 169 | \if{html}{\out{}} 170 | \if{latex}{\out{\hypertarget{method-MutationReporter-end_file}{}}} 171 | \subsection{Method \code{end_file()}}{ 172 | End testing current file 173 | \subsection{Usage}{ 174 | \if{html}{\out{
}}\preformatted{MutationReporter$end_file()}\if{html}{\out{
}} 175 | } 176 | 177 | } 178 | \if{html}{\out{
}} 179 | \if{html}{\out{}} 180 | \if{latex}{\out{\hypertarget{method-MutationReporter-end_reporter}{}}} 181 | \subsection{Method \code{end_reporter()}}{ 182 | End reporter and show summary 183 | \subsection{Usage}{ 184 | \if{html}{\out{
}}\preformatted{MutationReporter$end_reporter()}\if{html}{\out{
}} 185 | } 186 | 187 | } 188 | \if{html}{\out{
}} 189 | \if{html}{\out{}} 190 | \if{latex}{\out{\hypertarget{method-MutationReporter-get_score}{}}} 191 | \subsection{Method \code{get_score()}}{ 192 | Get the current score 193 | \subsection{Usage}{ 194 | \if{html}{\out{
}}\preformatted{MutationReporter$get_score()}\if{html}{\out{
}} 195 | } 196 | 197 | } 198 | \if{html}{\out{
}} 199 | \if{html}{\out{}} 200 | \if{latex}{\out{\hypertarget{method-MutationReporter-cat_tight}{}}} 201 | \subsection{Method \code{cat_tight()}}{ 202 | Print a message to the output 203 | \subsection{Usage}{ 204 | \if{html}{\out{
}}\preformatted{MutationReporter$cat_tight(...)}\if{html}{\out{
}} 205 | } 206 | 207 | \subsection{Arguments}{ 208 | \if{html}{\out{
}} 209 | \describe{ 210 | \item{\code{...}}{Message to print} 211 | } 212 | \if{html}{\out{
}} 213 | } 214 | } 215 | \if{html}{\out{
}} 216 | \if{html}{\out{}} 217 | \if{latex}{\out{\hypertarget{method-MutationReporter-cat_line}{}}} 218 | \subsection{Method \code{cat_line()}}{ 219 | Print a message to the output 220 | \subsection{Usage}{ 221 | \if{html}{\out{
}}\preformatted{MutationReporter$cat_line(...)}\if{html}{\out{
}} 222 | } 223 | 224 | \subsection{Arguments}{ 225 | \if{html}{\out{
}} 226 | \describe{ 227 | \item{\code{...}}{Message to print} 228 | } 229 | \if{html}{\out{
}} 230 | } 231 | } 232 | \if{html}{\out{
}} 233 | \if{html}{\out{}} 234 | \if{latex}{\out{\hypertarget{method-MutationReporter-rule}{}}} 235 | \subsection{Method \code{rule()}}{ 236 | Print a message to the output with a rule 237 | \subsection{Usage}{ 238 | \if{html}{\out{
}}\preformatted{MutationReporter$rule(...)}\if{html}{\out{
}} 239 | } 240 | 241 | \subsection{Arguments}{ 242 | \if{html}{\out{
}} 243 | \describe{ 244 | \item{\code{...}}{Message to print} 245 | } 246 | \if{html}{\out{
}} 247 | } 248 | } 249 | \if{html}{\out{
}} 250 | \if{html}{\out{}} 251 | \if{latex}{\out{\hypertarget{method-MutationReporter-clone}{}}} 252 | \subsection{Method \code{clone()}}{ 253 | The objects of this class are cloneable with this method. 254 | \subsection{Usage}{ 255 | \if{html}{\out{
}}\preformatted{MutationReporter$clone(deep = FALSE)}\if{html}{\out{
}} 256 | } 257 | 258 | \subsection{Arguments}{ 259 | \if{html}{\out{
}} 260 | \describe{ 261 | \item{\code{deep}}{Whether to make a deep clone.} 262 | } 263 | \if{html}{\out{
}} 264 | } 265 | } 266 | } 267 | -------------------------------------------------------------------------------- /man/PackageCopyStrategy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/project_copy_strategy.R 3 | \name{PackageCopyStrategy} 4 | \alias{PackageCopyStrategy} 5 | \title{Package copy strategy} 6 | \description{ 7 | It copies all files and directories from the original directory to a temporary directory. 8 | } 9 | \seealso{ 10 | Other CopyStrategy: 11 | \code{\link{CopyStrategy}}, 12 | \code{\link{default_copy_strategy}()} 13 | } 14 | \concept{CopyStrategy} 15 | \section{Super class}{ 16 | \code{\link[muttest:CopyStrategy]{muttest::CopyStrategy}} -> \code{PackageCopyStrategy} 17 | } 18 | \section{Methods}{ 19 | \subsection{Public methods}{ 20 | \itemize{ 21 | \item \href{#method-PackageCopyStrategy-execute}{\code{PackageCopyStrategy$execute()}} 22 | \item \href{#method-PackageCopyStrategy-clone}{\code{PackageCopyStrategy$clone()}} 23 | } 24 | } 25 | \if{html}{\out{
}} 26 | \if{html}{\out{}} 27 | \if{latex}{\out{\hypertarget{method-PackageCopyStrategy-execute}{}}} 28 | \subsection{Method \code{execute()}}{ 29 | Copy project files, excluding hidden and temp directories 30 | \subsection{Usage}{ 31 | \if{html}{\out{
}}\preformatted{PackageCopyStrategy$execute(original_dir, plan)}\if{html}{\out{
}} 32 | } 33 | 34 | \subsection{Arguments}{ 35 | \if{html}{\out{
}} 36 | \describe{ 37 | \item{\code{original_dir}}{The original directory to copy from} 38 | 39 | \item{\code{plan}}{The current test plan} 40 | } 41 | \if{html}{\out{
}} 42 | } 43 | \subsection{Returns}{ 44 | The path to the temporary directory 45 | } 46 | } 47 | \if{html}{\out{
}} 48 | \if{html}{\out{}} 49 | \if{latex}{\out{\hypertarget{method-PackageCopyStrategy-clone}{}}} 50 | \subsection{Method \code{clone()}}{ 51 | The objects of this class are cloneable with this method. 52 | \subsection{Usage}{ 53 | \if{html}{\out{
}}\preformatted{PackageCopyStrategy$clone(deep = FALSE)}\if{html}{\out{
}} 54 | } 55 | 56 | \subsection{Arguments}{ 57 | \if{html}{\out{
}} 58 | \describe{ 59 | \item{\code{deep}}{Whether to make a deep clone.} 60 | } 61 | \if{html}{\out{
}} 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /man/ProgressMutationReporter.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/reporter-progress.R 3 | \name{ProgressMutationReporter} 4 | \alias{ProgressMutationReporter} 5 | \title{Progress Reporter for Mutation Testing} 6 | \description{ 7 | A reporter that displays a progress indicator for mutation tests. 8 | It provides real-time feedback on which mutants are being tested and whether they were killed by tests. 9 | } 10 | \seealso{ 11 | Other MutationReporter: 12 | \code{\link{MutationReporter}}, 13 | \code{\link{default_reporter}()} 14 | } 15 | \concept{MutationReporter} 16 | \section{Super class}{ 17 | \code{\link[muttest:MutationReporter]{muttest::MutationReporter}} -> \code{ProgressMutationReporter} 18 | } 19 | \section{Public fields}{ 20 | \if{html}{\out{
}} 21 | \describe{ 22 | \item{\code{start_time}}{Time when testing started (for duration calculation)} 23 | 24 | \item{\code{min_time}}{Minimum test duration to display timing information} 25 | 26 | \item{\code{col_config}}{List of column configuration for report formatting} 27 | } 28 | \if{html}{\out{
}} 29 | } 30 | \section{Methods}{ 31 | \subsection{Public methods}{ 32 | \itemize{ 33 | \item \href{#method-ProgressMutationReporter-format_column}{\code{ProgressMutationReporter$format_column()}} 34 | \item \href{#method-ProgressMutationReporter-fmt_h}{\code{ProgressMutationReporter$fmt_h()}} 35 | \item \href{#method-ProgressMutationReporter-fmt_r}{\code{ProgressMutationReporter$fmt_r()}} 36 | \item \href{#method-ProgressMutationReporter-new}{\code{ProgressMutationReporter$new()}} 37 | \item \href{#method-ProgressMutationReporter-start_reporter}{\code{ProgressMutationReporter$start_reporter()}} 38 | \item \href{#method-ProgressMutationReporter-add_result}{\code{ProgressMutationReporter$add_result()}} 39 | \item \href{#method-ProgressMutationReporter-update}{\code{ProgressMutationReporter$update()}} 40 | \item \href{#method-ProgressMutationReporter-end_file}{\code{ProgressMutationReporter$end_file()}} 41 | \item \href{#method-ProgressMutationReporter-cr}{\code{ProgressMutationReporter$cr()}} 42 | \item \href{#method-ProgressMutationReporter-end_reporter}{\code{ProgressMutationReporter$end_reporter()}} 43 | \item \href{#method-ProgressMutationReporter-clone}{\code{ProgressMutationReporter$clone()}} 44 | } 45 | } 46 | \if{html}{\out{ 47 |
Inherited methods 48 | 57 |
58 | }} 59 | \if{html}{\out{
}} 60 | \if{html}{\out{}} 61 | \if{latex}{\out{\hypertarget{method-ProgressMutationReporter-format_column}{}}} 62 | \subsection{Method \code{format_column()}}{ 63 | Format a column with specified padding and width 64 | \subsection{Usage}{ 65 | \if{html}{\out{
}}\preformatted{ProgressMutationReporter$format_column(text, col_name, colorize = NULL)}\if{html}{\out{
}} 66 | } 67 | 68 | \subsection{Arguments}{ 69 | \if{html}{\out{
}} 70 | \describe{ 71 | \item{\code{text}}{Text to format} 72 | 73 | \item{\code{col_name}}{Column name to use configuration from} 74 | 75 | \item{\code{colorize}}{Optional function to color the text} 76 | } 77 | \if{html}{\out{
}} 78 | } 79 | } 80 | \if{html}{\out{
}} 81 | \if{html}{\out{}} 82 | \if{latex}{\out{\hypertarget{method-ProgressMutationReporter-fmt_h}{}}} 83 | \subsection{Method \code{fmt_h()}}{ 84 | Format the header of the report 85 | \subsection{Usage}{ 86 | \if{html}{\out{
}}\preformatted{ProgressMutationReporter$fmt_h()}\if{html}{\out{
}} 87 | } 88 | 89 | } 90 | \if{html}{\out{
}} 91 | \if{html}{\out{}} 92 | \if{latex}{\out{\hypertarget{method-ProgressMutationReporter-fmt_r}{}}} 93 | \subsection{Method \code{fmt_r()}}{ 94 | Format a row of the report 95 | \subsection{Usage}{ 96 | \if{html}{\out{
}}\preformatted{ProgressMutationReporter$fmt_r(status, k, s, e, t, score, mutator, file)}\if{html}{\out{
}} 97 | } 98 | 99 | \subsection{Arguments}{ 100 | \if{html}{\out{
}} 101 | \describe{ 102 | \item{\code{status}}{Status symbol (e.g., tick or cross)} 103 | 104 | \item{\code{k}}{Number of killed mutations} 105 | 106 | \item{\code{s}}{Number of survived mutations} 107 | 108 | \item{\code{e}}{Number of errors} 109 | 110 | \item{\code{t}}{Total number of mutations} 111 | 112 | \item{\code{score}}{Score percentage} 113 | 114 | \item{\code{mutator}}{The mutator used} 115 | 116 | \item{\code{file}}{The file being tested} 117 | } 118 | \if{html}{\out{
}} 119 | } 120 | \subsection{Returns}{ 121 | Formatted row string 122 | } 123 | } 124 | \if{html}{\out{
}} 125 | \if{html}{\out{}} 126 | \if{latex}{\out{\hypertarget{method-ProgressMutationReporter-new}{}}} 127 | \subsection{Method \code{new()}}{ 128 | Initialize a new progress reporter 129 | \subsection{Usage}{ 130 | \if{html}{\out{
}}\preformatted{ProgressMutationReporter$new( 131 | test_reporter = "silent", 132 | min_time = 1, 133 | file = stdout() 134 | )}\if{html}{\out{
}} 135 | } 136 | 137 | \subsection{Arguments}{ 138 | \if{html}{\out{
}} 139 | \describe{ 140 | \item{\code{test_reporter}}{Reporter to use for testthat::test_dir} 141 | 142 | \item{\code{min_time}}{Minimum time to show elapsed time (default: 1s)} 143 | 144 | \item{\code{file}}{Output destination (default: stdout)} 145 | } 146 | \if{html}{\out{
}} 147 | } 148 | } 149 | \if{html}{\out{
}} 150 | \if{html}{\out{}} 151 | \if{latex}{\out{\hypertarget{method-ProgressMutationReporter-start_reporter}{}}} 152 | \subsection{Method \code{start_reporter()}}{ 153 | Start reporter 154 | \subsection{Usage}{ 155 | \if{html}{\out{
}}\preformatted{ProgressMutationReporter$start_reporter(plan = NULL)}\if{html}{\out{
}} 156 | } 157 | 158 | \subsection{Arguments}{ 159 | \if{html}{\out{
}} 160 | \describe{ 161 | \item{\code{plan}}{The complete mutation plan} 162 | } 163 | \if{html}{\out{
}} 164 | } 165 | } 166 | \if{html}{\out{
}} 167 | \if{html}{\out{}} 168 | \if{latex}{\out{\hypertarget{method-ProgressMutationReporter-add_result}{}}} 169 | \subsection{Method \code{add_result()}}{ 170 | Add a mutation test result 171 | \subsection{Usage}{ 172 | \if{html}{\out{
}}\preformatted{ProgressMutationReporter$add_result(plan, killed, survived, errors)}\if{html}{\out{
}} 173 | } 174 | 175 | \subsection{Arguments}{ 176 | \if{html}{\out{
}} 177 | \describe{ 178 | \item{\code{plan}}{Current testing plan. See \code{plan()}.} 179 | 180 | \item{\code{killed}}{Whether the mutation was killed by tests} 181 | 182 | \item{\code{survived}}{Number of survived mutations} 183 | 184 | \item{\code{errors}}{Number of errors encountered} 185 | } 186 | \if{html}{\out{
}} 187 | } 188 | } 189 | \if{html}{\out{
}} 190 | \if{html}{\out{}} 191 | \if{latex}{\out{\hypertarget{method-ProgressMutationReporter-update}{}}} 192 | \subsection{Method \code{update()}}{ 193 | Update status spinner (for long-running operations) 194 | \subsection{Usage}{ 195 | \if{html}{\out{
}}\preformatted{ProgressMutationReporter$update(force = FALSE)}\if{html}{\out{
}} 196 | } 197 | 198 | \subsection{Arguments}{ 199 | \if{html}{\out{
}} 200 | \describe{ 201 | \item{\code{force}}{Force update even if interval hasn't elapsed} 202 | } 203 | \if{html}{\out{
}} 204 | } 205 | } 206 | \if{html}{\out{
}} 207 | \if{html}{\out{}} 208 | \if{latex}{\out{\hypertarget{method-ProgressMutationReporter-end_file}{}}} 209 | \subsection{Method \code{end_file()}}{ 210 | End testing current file 211 | \subsection{Usage}{ 212 | \if{html}{\out{
}}\preformatted{ProgressMutationReporter$end_file()}\if{html}{\out{
}} 213 | } 214 | 215 | } 216 | \if{html}{\out{
}} 217 | \if{html}{\out{}} 218 | \if{latex}{\out{\hypertarget{method-ProgressMutationReporter-cr}{}}} 219 | \subsection{Method \code{cr()}}{ 220 | Carriage return if dynamic, newline otherwise 221 | \subsection{Usage}{ 222 | \if{html}{\out{
}}\preformatted{ProgressMutationReporter$cr()}\if{html}{\out{
}} 223 | } 224 | 225 | } 226 | \if{html}{\out{
}} 227 | \if{html}{\out{}} 228 | \if{latex}{\out{\hypertarget{method-ProgressMutationReporter-end_reporter}{}}} 229 | \subsection{Method \code{end_reporter()}}{ 230 | End reporter with detailed summary 231 | \subsection{Usage}{ 232 | \if{html}{\out{
}}\preformatted{ProgressMutationReporter$end_reporter()}\if{html}{\out{
}} 233 | } 234 | 235 | } 236 | \if{html}{\out{
}} 237 | \if{html}{\out{}} 238 | \if{latex}{\out{\hypertarget{method-ProgressMutationReporter-clone}{}}} 239 | \subsection{Method \code{clone()}}{ 240 | The objects of this class are cloneable with this method. 241 | \subsection{Usage}{ 242 | \if{html}{\out{
}}\preformatted{ProgressMutationReporter$clone(deep = FALSE)}\if{html}{\out{
}} 243 | } 244 | 245 | \subsection{Arguments}{ 246 | \if{html}{\out{
}} 247 | \describe{ 248 | \item{\code{deep}}{Whether to make a deep clone.} 249 | } 250 | \if{html}{\out{
}} 251 | } 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /man/TestStrategy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/test_strategy.R 3 | \name{TestStrategy} 4 | \alias{TestStrategy} 5 | \title{TestStrategy interface} 6 | \description{ 7 | Extend this class to implement a custom test strategy. 8 | } 9 | \seealso{ 10 | Other TestStrategy: 11 | \code{\link{FileTestStrategy}}, 12 | \code{\link{FullTestStrategy}}, 13 | \code{\link{default_test_strategy}()} 14 | } 15 | \concept{TestStrategy} 16 | \section{Methods}{ 17 | \subsection{Public methods}{ 18 | \itemize{ 19 | \item \href{#method-TestStrategy-execute}{\code{TestStrategy$execute()}} 20 | \item \href{#method-TestStrategy-clone}{\code{TestStrategy$clone()}} 21 | } 22 | } 23 | \if{html}{\out{
}} 24 | \if{html}{\out{}} 25 | \if{latex}{\out{\hypertarget{method-TestStrategy-execute}{}}} 26 | \subsection{Method \code{execute()}}{ 27 | Execute the test strategy 28 | \subsection{Usage}{ 29 | \if{html}{\out{
}}\preformatted{TestStrategy$execute(path, plan, reporter)}\if{html}{\out{
}} 30 | } 31 | 32 | \subsection{Arguments}{ 33 | \if{html}{\out{
}} 34 | \describe{ 35 | \item{\code{path}}{The path to the test directory} 36 | 37 | \item{\code{plan}}{The current mutation plan. See \code{plan()}.} 38 | 39 | \item{\code{reporter}}{The reporter to use for test results} 40 | } 41 | \if{html}{\out{
}} 42 | } 43 | \subsection{Returns}{ 44 | The test result 45 | } 46 | } 47 | \if{html}{\out{
}} 48 | \if{html}{\out{}} 49 | \if{latex}{\out{\hypertarget{method-TestStrategy-clone}{}}} 50 | \subsection{Method \code{clone()}}{ 51 | The objects of this class are cloneable with this method. 52 | \subsection{Usage}{ 53 | \if{html}{\out{
}}\preformatted{TestStrategy$clone(deep = FALSE)}\if{html}{\out{
}} 54 | } 55 | 56 | \subsection{Arguments}{ 57 | \if{html}{\out{
}} 58 | \describe{ 59 | \item{\code{deep}}{Whether to make a deep clone.} 60 | } 61 | \if{html}{\out{
}} 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /man/default_copy_strategy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/project_copy_strategy.R 3 | \name{default_copy_strategy} 4 | \alias{default_copy_strategy} 5 | \title{Create a default project copy strategy} 6 | \usage{ 7 | default_copy_strategy(...) 8 | } 9 | \arguments{ 10 | \item{...}{Arguments passed to the \code{?PackageCopyStrategy} constructor.} 11 | } 12 | \value{ 13 | A \code{?CopyStrategy} object 14 | } 15 | \description{ 16 | Create a default project copy strategy 17 | } 18 | \seealso{ 19 | Other CopyStrategy: 20 | \code{\link{CopyStrategy}}, 21 | \code{\link{PackageCopyStrategy}} 22 | } 23 | \concept{CopyStrategy} 24 | -------------------------------------------------------------------------------- /man/default_reporter.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/reporter.R 3 | \name{default_reporter} 4 | \alias{default_reporter} 5 | \title{Create a default reporter} 6 | \usage{ 7 | default_reporter(...) 8 | } 9 | \arguments{ 10 | \item{...}{Arguments passed to the \code{?ProgressMutationReporter} constructor.} 11 | } 12 | \description{ 13 | Create a default reporter 14 | } 15 | \seealso{ 16 | Other MutationReporter: 17 | \code{\link{MutationReporter}}, 18 | \code{\link{ProgressMutationReporter}} 19 | } 20 | \concept{MutationReporter} 21 | -------------------------------------------------------------------------------- /man/default_test_strategy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/test_strategy.R 3 | \name{default_test_strategy} 4 | \alias{default_test_strategy} 5 | \title{Create a default run strategy} 6 | \usage{ 7 | default_test_strategy(...) 8 | } 9 | \arguments{ 10 | \item{...}{Arguments passed to the \code{?FullTestStrategy} constructor.} 11 | } 12 | \value{ 13 | A \code{?TestStrategy} object 14 | } 15 | \description{ 16 | Create a default run strategy 17 | } 18 | \seealso{ 19 | Other TestStrategy: 20 | \code{\link{FileTestStrategy}}, 21 | \code{\link{FullTestStrategy}}, 22 | \code{\link{TestStrategy}} 23 | } 24 | \concept{TestStrategy} 25 | -------------------------------------------------------------------------------- /man/figures/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jakubsob/muttest/b2307fc406d1941a83d9621ff556697193262236/man/figures/logo.png -------------------------------------------------------------------------------- /man/muttest.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/muttest.R 3 | \name{muttest} 4 | \alias{muttest} 5 | \title{Run a mutation test} 6 | \usage{ 7 | muttest( 8 | plan, 9 | path = "tests/testthat", 10 | reporter = default_reporter(), 11 | test_strategy = default_test_strategy(), 12 | copy_strategy = default_copy_strategy() 13 | ) 14 | } 15 | \arguments{ 16 | \item{plan}{A data frame with the test plan. See \code{plan()}.} 17 | 18 | \item{path}{Path to the test directory.} 19 | 20 | \item{reporter}{Reporter to use for mutation testing results. See \code{?MutationReporter}.} 21 | 22 | \item{test_strategy}{Strategy for running tests. See \code{?TestStrategy}. 23 | The purpose of test strategy is to control how tests are executed. 24 | We can run all tests for each mutant, or only tests that are relevant to the mutant.} 25 | 26 | \item{copy_strategy}{Strategy for copying the project. See \code{?CopyStrategy}. 27 | This strategy controls which files are copied to the temporary directory, where the tests are run.} 28 | } 29 | \value{ 30 | A numeric value representing the mutation score. 31 | } 32 | \description{ 33 | Run a mutation test 34 | } 35 | -------------------------------------------------------------------------------- /man/operator.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/mutator-operator.R 3 | \name{operator} 4 | \alias{operator} 5 | \title{Mutate an operator} 6 | \usage{ 7 | operator(from, to) 8 | } 9 | \arguments{ 10 | \item{from}{The operator to be replaced.} 11 | 12 | \item{to}{The operator to replace with.} 13 | } 14 | \description{ 15 | It changes a binary operator to another one. 16 | } 17 | \examples{ 18 | operator("==", "!=") 19 | operator(">", "<") 20 | operator("<", ">") 21 | operator("+", "-") 22 | 23 | } 24 | -------------------------------------------------------------------------------- /man/plan.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/muttest.R 3 | \name{plan} 4 | \alias{plan} 5 | \title{Create a plan for mutation testing} 6 | \usage{ 7 | plan(mutators, source_files = fs::dir_ls("R", regexp = ".[rR]$")) 8 | } 9 | \arguments{ 10 | \item{mutators}{A list of mutators to use. See \code{\link[=operator]{operator()}}.} 11 | 12 | \item{source_files}{A vector of file paths to the source files.} 13 | } 14 | \value{ 15 | A data frame with the test plan. 16 | The data frame has the following columns: 17 | \itemize{ 18 | \item \code{filename}: The name of the source file. 19 | \item \code{original_code}: The original code of the source file. 20 | \item \code{mutated_code}: The mutated code of the source file. 21 | \item \code{mutator}: The mutator that was applied. 22 | } 23 | } 24 | \description{ 25 | Each mutant requires rerunning the tests. For large project it might be not feasible to test all 26 | mutants in one go. This function allows you to create a plan for selected source files and mutators. 27 | } 28 | \details{ 29 | The plan is in a data frame format, where each row represents a mutant. 30 | 31 | You can subset the plan before passing it to the \code{muttest()} function. 32 | } 33 | -------------------------------------------------------------------------------- /muttest.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | ProjectId: 4c6babe0-248e-427a-8f3b-5a7622f0b0f0 3 | 4 | RestoreWorkspace: Default 5 | SaveWorkspace: Default 6 | AlwaysSaveHistory: Default 7 | 8 | EnableCodeIndexing: Yes 9 | UseSpacesForTab: Yes 10 | NumSpacesForTab: 2 11 | Encoding: UTF-8 12 | 13 | RnwWeave: Sweave 14 | LaTeX: pdfLaTeX 15 | 16 | AutoAppendNewline: Yes 17 | StripTrailingWhitespace: Yes 18 | 19 | BuildType: Package 20 | PackageUseDevtools: Yes 21 | PackageInstallArgs: --no-multiarch --with-keep.source 22 | -------------------------------------------------------------------------------- /pkgdown/favicon/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jakubsob/muttest/b2307fc406d1941a83d9621ff556697193262236/pkgdown/favicon/apple-touch-icon.png -------------------------------------------------------------------------------- /pkgdown/favicon/favicon-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jakubsob/muttest/b2307fc406d1941a83d9621ff556697193262236/pkgdown/favicon/favicon-96x96.png -------------------------------------------------------------------------------- /pkgdown/favicon/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jakubsob/muttest/b2307fc406d1941a83d9621ff556697193262236/pkgdown/favicon/favicon.ico -------------------------------------------------------------------------------- /pkgdown/favicon/favicon.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkgdown/favicon/site.webmanifest: -------------------------------------------------------------------------------- 1 | { 2 | "name": "", 3 | "short_name": "", 4 | "icons": [ 5 | { 6 | "src": "/web-app-manifest-192x192.png", 7 | "sizes": "192x192", 8 | "type": "image/png", 9 | "purpose": "maskable" 10 | }, 11 | { 12 | "src": "/web-app-manifest-512x512.png", 13 | "sizes": "512x512", 14 | "type": "image/png", 15 | "purpose": "maskable" 16 | } 17 | ], 18 | "theme_color": "#ffffff", 19 | "background_color": "#ffffff", 20 | "display": "standalone" 21 | } -------------------------------------------------------------------------------- /pkgdown/favicon/web-app-manifest-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jakubsob/muttest/b2307fc406d1941a83d9621ff556697193262236/pkgdown/favicon/web-app-manifest-192x192.png -------------------------------------------------------------------------------- /pkgdown/favicon/web-app-manifest-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jakubsob/muttest/b2307fc406d1941a83d9621ff556697193262236/pkgdown/favicon/web-app-manifest-512x512.png -------------------------------------------------------------------------------- /tests/acceptance/setup-steps.R: -------------------------------------------------------------------------------- 1 | cucumber::before(function(context, scenario_name) { 2 | dir <- fs::path(tempdir(), "__muttest_tmp___") 3 | fs::dir_create(dir) 4 | context$dir <- dir 5 | }) 6 | 7 | cucumber::given( 8 | "I have a {string} file with", 9 | function(filename, code, context) { 10 | file <- fs::path(context$dir, filename) 11 | fs::dir_create(fs::path_dir(file)) 12 | writeLines(code, file) 13 | } 14 | ) 15 | 16 | cucumber::given("I clone a repository from {string}", function(url, context) { 17 | path <- get_repo(url) 18 | context$dir <- path 19 | }) 20 | 21 | cucumber::when("I run mutation tests with", function(code, context) { 22 | withr::with_output_sink(new = nullfile(), { 23 | withr::with_dir(context$dir, { 24 | suppressPackageStartupMessages( 25 | context$score <- eval(parse(text = code)) 26 | ) 27 | }) 28 | }) 29 | }) 30 | 31 | cucumber::then("the mutation score should be {float}", function(score, context) { 32 | expect_equal(context$score, score) 33 | }) 34 | 35 | cucumber::then("the mutation score should be {word}", function(score, context) { 36 | if (score == "NA") { 37 | score <- NA_real_ 38 | } 39 | expect_equal(context$score, score) 40 | }) 41 | 42 | cucumber::then("I get a mutation score", function(context) { 43 | expect_true(!is.na(context$score)) 44 | }) 45 | 46 | cucumber::after(function(context, scenario_name) { 47 | fs::dir_delete(context$dir) 48 | }) 49 | -------------------------------------------------------------------------------- /tests/acceptance/setup.R: -------------------------------------------------------------------------------- 1 | get_repo <- function(url) { 2 | parts <- strsplit(url, "/")[[1]] 3 | zip <- parts[length(parts)] 4 | withr::with_dir(fs::path(tempdir()), { 5 | tryCatch( 6 | download.file(url = url, destfile = zip, quiet = TRUE), 7 | error = function(e) { 8 | testthat::skip(paste("Failed to download repository from", url)) 9 | } 10 | ) 11 | path <- unzip(zipfile = zip) 12 | }) 13 | fs::path(tempdir(), fs::path_common(path)) 14 | } 15 | -------------------------------------------------------------------------------- /tests/acceptance/test_box.feature: -------------------------------------------------------------------------------- 1 | Feature: Test klmr::box project 2 | 3 | This is a specification of how mutation testing of a project using klmr::box works. 4 | 5 | Scenario: Run plan with one source file and one mutation 6 | Given I have a "src/calculate.R" file with 7 | """ 8 | #' @export 9 | calculate <- function(x) { 10 | x + 1 11 | } 12 | """ 13 | And I have a "tests/test-calculate.R" file with 14 | """ 15 | box::use(../src/calculate[calculate]) 16 | 17 | test_that("calculate adds 1", { 18 | expect_equal(calculate(1), 2) 19 | }) 20 | """ 21 | When I run mutation tests with 22 | """ 23 | muttest( 24 | plan = plan( 25 | mutators = list( 26 | operator("+", "-") 27 | ), 28 | source_files = c("src/calculate.R") 29 | ), 30 | path = "tests", 31 | test_strategy = FileTestStrategy$new(load_package = "none") 32 | ) 33 | """ 34 | Then the mutation score should be 1.0 35 | 36 | Scenario: Function imported to a test file with the same name 37 | Given I have a "src/calculate.R" file with 38 | """ 39 | #' @export 40 | calculate <- function(x) { 41 | x + 1 42 | } 43 | """ 44 | Given I have a "src/calculate2.R" file with 45 | """ 46 | #' @export 47 | calculate <- function(x) { 48 | x * 1 49 | } 50 | """ 51 | And I have a "tests/test-calculate.R" file with 52 | """ 53 | box::use(../src/calculate[calculate]) 54 | 55 | test_that("calculate adds 1", { 56 | expect_equal(calculate(1), 2) 57 | }) 58 | """ 59 | And I have a "tests/test-calculate2.R" file with 60 | """ 61 | box::use(../src/calculate2[calculate]) 62 | 63 | test_that("calculate multiplies by 1", { 64 | expect_equal(calculate(1), 1) 65 | }) 66 | """ 67 | When I run mutation tests with 68 | """ 69 | plan <- plan( 70 | mutators = list( 71 | operator("+", "-"), 72 | operator("*", "/") 73 | ), 74 | source_files = c("src/calculate.R", "src/calculate2.R") 75 | ) 76 | muttest( 77 | plan = plan, 78 | path = "tests", 79 | test_strategy = FileTestStrategy$new(load_package = "none") 80 | ) 81 | """ 82 | Then the mutation score should be 0.5 83 | -------------------------------------------------------------------------------- /tests/acceptance/test_package.feature: -------------------------------------------------------------------------------- 1 | Feature: Test package 2 | 3 | This is a specification of how mutation testing of a package works. 4 | It is the default behavior, requiring the least amount of configuration to run tests. 5 | 6 | Scenario: Run plan with one source file and one mutation 7 | Given I have a "DESCRIPTION" file with 8 | """ 9 | Package: example 10 | Version: 0.1.0 11 | """ 12 | Given I have a "R/calculate.R" file with 13 | """ 14 | calculate <- function(x) { 15 | x + 1 16 | } 17 | """ 18 | And I have a "tests/testthat/test-calculate.R" file with 19 | """ 20 | test_that("calculate adds 1", { 21 | expect_equal(calculate(1), 2) 22 | }) 23 | """ 24 | When I run mutation tests with 25 | """ 26 | muttest( 27 | path = "tests/testthat", 28 | plan = plan( 29 | mutators = list( 30 | operator("+", "-") 31 | ) 32 | ) 33 | ) 34 | """ 35 | Then the mutation score should be 1.0 36 | 37 | Scenario: Run plan with one source file and multiple mutations 38 | Given I have a "DESCRIPTION" file with 39 | """ 40 | Package: example 41 | Version: 0.1.0 42 | """ 43 | Given I have a "R/calculate.R" file with 44 | """ 45 | calculate <- function(x) { 46 | x + 1 / 2 47 | } 48 | """ 49 | And I have a "tests/testthat/test-calculate.R" file with 50 | """ 51 | test_that("calculate adds 1", { 52 | expect_equal(calculate(1), 2) 53 | }) 54 | """ 55 | When I run mutation tests with 56 | """ 57 | muttest( 58 | path = "tests/testthat", 59 | plan = plan( 60 | mutators = list( 61 | operator("+", "-"), 62 | operator("/", "*") 63 | ) 64 | ) 65 | ) 66 | """ 67 | Then the mutation score should be 1.0 68 | 69 | Scenario: Run plan with multiple source files and multiple mutations 70 | Given I have a "DESCRIPTION" file with 71 | """ 72 | Package: example 73 | Version: 0.1.0 74 | """ 75 | Given I have a "R/calculate.R" file with 76 | """ 77 | calculate <- function(x) { 78 | x + 1 79 | } 80 | """ 81 | And I have a "R/another_file.R" file with 82 | """ 83 | another_function <- function(x) { 84 | x * 2 85 | } 86 | """ 87 | And I have a "tests/testthat/test-calculate.R" file with 88 | """ 89 | test_that("calculate adds 1", { 90 | expect_equal(calculate(1), 2) 91 | }) 92 | """ 93 | And I have a "tests/testthat/test-another_file.R" file with 94 | """ 95 | test_that("another_function multiplies by 2", { 96 | expect_equal(another_function(2), 4) 97 | }) 98 | """ 99 | When I run mutation tests with 100 | """ 101 | muttest( 102 | path = "tests/testthat", 103 | plan = plan( 104 | mutators = list( 105 | operator("+", "-"), 106 | operator("/", "*") 107 | ) 108 | ) 109 | ) 110 | """ 111 | Then the mutation score should be 1.0 112 | 113 | Scenario: Run plan with no relevant mutations 114 | 115 | If no mutations can be applied to the code, the mutation score is NA. 116 | 117 | Given I have a "DESCRIPTION" file with 118 | """ 119 | Package: example 120 | Version: 0.1.0 121 | """ 122 | Given I have a "R/calculate.R" file with 123 | """ 124 | calculate <- function(x) { 125 | x + 1 126 | } 127 | """ 128 | And I have a "tests/testthat/test-calculate.R" file with 129 | """ 130 | test_that("calculate adds 1", { 131 | expect_equal(calculate(1), 2) 132 | }) 133 | """ 134 | When I run mutation tests with 135 | """ 136 | muttest( 137 | path = "tests/testthat", 138 | plan = plan( 139 | mutators = list( 140 | operator("*", "/") 141 | ) 142 | ) 143 | ) 144 | """ 145 | Then the mutation score should be NA 146 | 147 | Scenario: Test runs with errors don't count in the mutation score 148 | 149 | Mutating "+" to "-" triggers an error in the function and the assertion doesn't pass. 150 | It's not a failute of a test, but a failure of the function. 151 | This mutation is not counted in the mutation score. 152 | Only change from "+" to "*" is counted. 153 | There are 2 mutations, 1 error, 1 killed, score is 50%. 154 | 155 | Given I have a "DESCRIPTION" file with 156 | """ 157 | Package: example 158 | Version: 0.1.0 159 | """ 160 | Given I have a "R/calculate.R" file with 161 | """ 162 | calculate <- function(x) { 163 | score <- x + 1 164 | if (score == 0) { 165 | stop("Score is zero") 166 | } 167 | score 168 | } 169 | """ 170 | And I have a "tests/testthat/test-calculate.R" file with 171 | """ 172 | test_that("calculate adds 1", { 173 | expect_equal(calculate(1), 2) 174 | }) 175 | """ 176 | When I run mutation tests with 177 | """ 178 | muttest( 179 | path = "tests/testthat", 180 | plan = plan( 181 | mutators = list( 182 | operator("+", "-"), 183 | operator("+", "*") 184 | ) 185 | ) 186 | ) 187 | """ 188 | Then the mutation score should be 0.5 189 | 190 | Scenario: Test runs with only errors 191 | There are 2 mutations, 2 errors, 0 killed, score is 0%. 192 | 193 | Given I have a "DESCRIPTION" file with 194 | """ 195 | Package: example 196 | Version: 0.1.0 197 | """ 198 | Given I have a "R/calculate.R" file with 199 | """ 200 | calculate <- function(x) { 201 | score <- x + 1 202 | stop("Score is zero") 203 | score 204 | } 205 | """ 206 | And I have a "tests/testthat/test-calculate.R" file with 207 | """ 208 | test_that("calculate adds 1", { 209 | expect_equal(calculate(1), 2) 210 | }) 211 | """ 212 | When I run mutation tests with 213 | """ 214 | muttest( 215 | path = "tests/testthat", 216 | plan = plan( 217 | mutators = list( 218 | operator("+", "-"), 219 | operator("+", "*") 220 | ) 221 | ) 222 | ) 223 | """ 224 | Then the mutation score should be 0.0 225 | -------------------------------------------------------------------------------- /tests/acceptance/test_repo.feature: -------------------------------------------------------------------------------- 1 | Feature: Test real R package 2 | 3 | This specification ensures that {muttest} works for the few selected, popular packages. 4 | 5 | Scenario: Testing dplyr 6 | Given I clone a repository from "https://github.com/tidyverse/dplyr/archive/main.zip" 7 | When I run mutation tests with 8 | """ 9 | plan <- plan( 10 | mutators = list( 11 | operator("+", "-") 12 | ) 13 | ) 14 | muttest( 15 | path = "tests/testthat", 16 | plan = plan[1:10, ], 17 | test_strategy = FileTestStrategy$new() 18 | ) 19 | """ 20 | Then I get a mutation score 21 | 22 | Scenario: Testing ggplot2 23 | Given I clone a repository from "https://github.com/tidyverse/ggplot2/archive/main.zip" 24 | When I run mutation tests with 25 | """ 26 | plan <- plan( 27 | mutators = list( 28 | operator("+", "-") 29 | ) 30 | ) 31 | muttest( 32 | path = "tests/testthat", 33 | plan = plan[1:10, ], 34 | test_strategy = FileTestStrategy$new() 35 | ) 36 | """ 37 | Then I get a mutation score 38 | 39 | Scenario: Testing shiny 40 | Given I clone a repository from "https://github.com/rstudio/shiny/archive/main.zip" 41 | When I run mutation tests with 42 | """ 43 | plan <- plan( 44 | mutators = list( 45 | operator("==", "!=") 46 | ) 47 | ) 48 | muttest( 49 | path = "tests/testthat", 50 | plan = plan[1, ] 51 | ) 52 | """ 53 | Then I get a mutation score 54 | -------------------------------------------------------------------------------- /tests/testthat.R: -------------------------------------------------------------------------------- 1 | # This file is part of the standard setup for testthat. 2 | # It is recommended that you do not modify it. 3 | # 4 | # Where should you do additional test configuration? 5 | # Learn more about the roles of various files in: 6 | # * https://r-pkgs.org/testing-design.html#sec-tests-files-overview 7 | # * https://testthat.r-lib.org/articles/special-files.html 8 | 9 | library(testthat) 10 | library(muttest) 11 | 12 | test_check("muttest") 13 | -------------------------------------------------------------------------------- /tests/testthat/test-acceptance.R: -------------------------------------------------------------------------------- 1 | # Run acceptance tests only when calculating the coverage 2 | testthat::skip_if_not(covr::in_covr()) 3 | 4 | cucumber::test("../acceptance") 5 | -------------------------------------------------------------------------------- /tests/testthat/test-mutator-operator.R: -------------------------------------------------------------------------------- 1 | describe("operator", { 2 | it("should generate mutations for a single operator", { 3 | # Arrange 4 | code <- c("x <- 1 + 2") 5 | mutator <- operator("+", "-") 6 | 7 | # Act 8 | mutations <- mutator$mutate(code) 9 | 10 | # Assert 11 | expect_equal( 12 | mutations, 13 | list( 14 | c("x <- 1 - 2") 15 | ) 16 | ) 17 | }) 18 | 19 | it("should return NULL when no mutations are possible", { 20 | # Arrange 21 | code <- c("x <- 1 - 2") 22 | mutator <- operator("+", "-") 23 | 24 | # Act 25 | mutations <- mutator$mutate(code) 26 | 27 | # Assert 28 | expect_null(mutations) 29 | }) 30 | 31 | it("should generate multiple mutations for multiple occurrences", { 32 | # Arrange 33 | code <- c("x <- 1 + 2 + 3") 34 | mutator <- operator("+", "-") 35 | 36 | # Act 37 | mutations <- mutator$mutate(code) 38 | 39 | # Assert 40 | expect_equal( 41 | mutations, 42 | list( 43 | c("x <- 1 - 2 + 3"), 44 | c("x <- 1 + 2 - 3") 45 | ) 46 | ) 47 | }) 48 | 49 | it("should work for multiline code", { 50 | # Arrange 51 | code <- c("x <- 1 + 2", "y <- x + 3") 52 | mutator <- operator("+", "-") 53 | 54 | # Act 55 | mutations <- mutator$mutate(code) 56 | 57 | # Assert 58 | expect_equal( 59 | mutations, 60 | list( 61 | c("x <- 1 - 2", "y <- x + 3"), 62 | c("x <- 1 + 2", "y <- x - 3") 63 | ) 64 | ) 65 | }) 66 | }) 67 | -------------------------------------------------------------------------------- /tests/testthat/test-muttest.R: -------------------------------------------------------------------------------- 1 | .with_example_dir <- function(path, code) { 2 | withr::with_dir( 3 | system.file("examples", path, package = "muttest"), 4 | code 5 | ) 6 | } 7 | 8 | test_ <- function(...) { 9 | purrr::quietly(muttest)(...)$result 10 | } 11 | 12 | test_that("operators", { 13 | .with_example_dir("operators/", { 14 | mutators <- list(operator("+", "-"), operator("*", "/")) 15 | plan <- plan(mutators, fs::dir_ls("R")) 16 | expect_equal( 17 | test_(plan), 18 | 0.5 19 | ) 20 | }) 21 | }) 22 | -------------------------------------------------------------------------------- /tests/testthat/test-project_copy_strategy.R: -------------------------------------------------------------------------------- 1 | describe("CopyStrategy", { 2 | it("throws error when calling execute", { 3 | # Arrange 4 | strategy <- CopyStrategy$new() 5 | 6 | # Act, Assert 7 | expect_error(strategy$execute("original_dir"), "Not implemented") 8 | }) 9 | }) 10 | 11 | describe("PackageCopyStrategy", { 12 | it("copies project files excluding hidden and temp directories", { 13 | # Arrange 14 | temp_dir <- withr::local_tempdir() 15 | original_dir <- file.path(temp_dir, "original") 16 | dir.create(original_dir) 17 | dir.create(file.path(original_dir, ".hidden")) 18 | dir.create(file.path(original_dir, "tmp")) 19 | dir.create(file.path(original_dir, "temp")) 20 | dir.create(file.path(original_dir, "inst")) 21 | dir.create(file.path(original_dir, "src")) 22 | 23 | strategy <- PackageCopyStrategy$new() 24 | 25 | # Act 26 | copied_dir <- strategy$execute(original_dir, data.frame()) 27 | 28 | # Assert 29 | expect_true(dir.exists(copied_dir)) 30 | expect_false(dir.exists(file.path(copied_dir, ".hidden"))) 31 | expect_false(dir.exists(file.path(copied_dir, "tmp"))) 32 | expect_false(dir.exists(file.path(copied_dir, "temp"))) 33 | expect_true(dir.exists(file.path(copied_dir, "src"))) 34 | expect_true(dir.exists(file.path(copied_dir, "inst"))) 35 | }) 36 | }) 37 | 38 | describe("default_copy_strategy", { 39 | it("returns a CopyStrategy object", { 40 | # Act 41 | strategy <- default_copy_strategy() 42 | 43 | # Assert 44 | expect_s3_class(strategy, "CopyStrategy") 45 | }) 46 | }) 47 | -------------------------------------------------------------------------------- /tests/testthat/test-test_strategy.R: -------------------------------------------------------------------------------- 1 | describe("TestStrategy", { 2 | it("throws error when calling execute()", { 3 | # Arrange 4 | strategy <- TestStrategy$new() 5 | 6 | # Act, Assert 7 | expect_error( 8 | strategy$execute("path", list(), NULL), 9 | "Not implemented" 10 | ) 11 | }) 12 | }) 13 | 14 | describe("FullTestStrategy", { 15 | it("executes all tests in the directory", { 16 | # Arrange 17 | temp_dir <- withr::local_tempdir() 18 | test_dir <- file.path(temp_dir, "tests/testthat") 19 | dir.create(test_dir, recursive = TRUE) 20 | test1 <- 'test_that("test1", { expect_true(TRUE) })' 21 | test2 <- 'test_that("test2", { expect_true(TRUE) })' 22 | writeLines(test1, file.path(test_dir, "test-file1.R")) 23 | writeLines(test2, file.path(test_dir, "test-file2.R")) 24 | strategy <- FullTestStrategy$new(load_package = "none") 25 | 26 | # Act 27 | result <- strategy$execute( 28 | path = test_dir, 29 | plan = list(mutated_file = "file.R"), 30 | reporter = testthat::SilentReporter$new() 31 | ) 32 | 33 | # Assert 34 | expect_equal(sum(as.data.frame(result)$passed), 2) 35 | }) 36 | }) 37 | 38 | describe("FileTestStrategy", { 39 | it("runs only tests matching the source file name", { 40 | # Arrange 41 | temp_dir <- withr::local_tempdir() 42 | test_dir <- file.path(temp_dir, "tests/testthat") 43 | dir.create(test_dir, recursive = TRUE) 44 | 45 | test1 <- 'test_that("test1", { expect_true(TRUE) })' 46 | test2 <- 'test_that("test2", { expect_true(TRUE) })' 47 | writeLines(test1, file.path(test_dir, "test-file1.R")) 48 | writeLines(test2, file.path(test_dir, "test-file2.R")) 49 | strategy <- FileTestStrategy$new(load_package = "none") 50 | 51 | # Act 52 | result <- strategy$execute( 53 | path = test_dir, 54 | plan = data.frame(filename = "file1.R"), 55 | reporter = testthat::SilentReporter$new() 56 | ) 57 | 58 | # Assert 59 | expect_equal(sum(as.data.frame(result)$passed), 1) 60 | }) 61 | 62 | it("doesn't run test files if source file name doesn't match", { 63 | # Arrange 64 | temp_dir <- withr::local_tempdir() 65 | test_dir <- file.path(temp_dir, "tests/testthat") 66 | dir.create(test_dir, recursive = TRUE) 67 | 68 | test1 <- 'test_that("test1", { expect_true(TRUE) })' 69 | test2 <- 'test_that("test2", { expect_true(TRUE) })' 70 | writeLines(test1, file.path(test_dir, "test-file1.R")) 71 | writeLines(test2, file.path(test_dir, "test-file2.R")) 72 | strategy <- FileTestStrategy$new(load_package = "none") 73 | 74 | # Act 75 | result <- strategy$execute( 76 | path = test_dir, 77 | plan = data.frame(filename = "file3.R"), 78 | reporter = testthat::SilentReporter$new() 79 | ) 80 | 81 | # Assert 82 | expect_equal(length(result), 0) 83 | }) 84 | }) 85 | 86 | 87 | test_that("default_test_strategy returns a FileTestStrategy", { 88 | strategy <- default_test_strategy() 89 | expect_s3_class(strategy, "R6") 90 | expect_true(inherits(strategy, "TestStrategy")) 91 | }) 92 | -------------------------------------------------------------------------------- /vignettes/articles/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | *.R 3 | -------------------------------------------------------------------------------- /vignettes/articles/how-it-works.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "How it works" 3 | output: rmarkdown::html_vignette 4 | vignette: > 5 | %\VignetteIndexEntry{Vignette Title} 6 | %\VignetteEngine{knitr::rmarkdown} 7 | %\VignetteEncoding{UTF-8} 8 | --- 9 | 10 | To get the mutation score, we need to do the following steps: 11 | 12 | # Generate mutants 13 | 14 | Project files are read and stored as character vectors by `test_plan`. 15 | `mutators` provided to `test_plan` are applied to the original code to generate mutants. 16 | 17 | If we have only one mutator `+` → `-` and code `a + b + c`, the generated mutants will be `a - b + c` and `a + b - c`. 18 | Even if we have only one mutator and one line of code, many mutants can be generated. 19 | 20 | Only one change like this is applied to the code at a time, one file at a time. 21 | 22 | # Copy the project and apply the mutation 23 | 24 | We copy the project to a temporary directory using `?CopyStrategy`. 25 | 26 | For each generated mutation, we create a fresh copy of the project, one copy is created at a time. 27 | 28 | We overwrite the mutated file in project copy. 29 | 30 | 31 | > 💡 This is needed since R is not a compiled language and code can be sourced during runtime. 32 | > 33 | > If the mutated code was evaluated from the lines we read and mutated in [step 1](#generate-mutants) (from memory), then we could miss some mutations, as some test files could source the original code. 34 | > 35 | > Even if we mutated the code from `R/calculate.R`, when running the test the original code would be sourced and the test would pass. 36 | > 37 | > ```r 38 | > #' testst/testthat/test_calculate.R 39 | > source("R/calculate.R") 40 | > test_that("calculate", { 41 | > expect_equal(calculate(1, 2), 3) 42 | > }) 43 | > ``` 44 | > 45 | > It's even more apparent for projects that use modules like `box`, they always source the tested code on the fly. 46 | > 47 | > ```r 48 | > #' testst/testthat/test_calculate.R 49 | > box::use(R/calculate) 50 | > test_that("calculate", { 51 | > expect_equal(calculate$calculate(1, 2), 3) 52 | > }) 53 | > ``` 54 | 55 | # Run tests and calculate the mutation score 56 | 57 | We run the tests on the copied project with the mutated code. 58 | 59 | The results are counted, only test failures and total number of test runs contribute to the mutation score. 60 | 61 | Some mutants will inevitably lead to runtime errors, those are not counted as failures and are not included in the mutation score. 62 | 63 | The mutation score is the percentage of mutants that were detected by the tests. 64 | 65 | # Clean up 66 | 67 | We clean up the temporary directory to remove any files that were created during the mutation testing process. 68 | --------------------------------------------------------------------------------