├── .Rbuildignore ├── .github ├── .gitignore ├── dependabot.yml └── workflows │ ├── check-standard.yaml │ └── pkgdown.yaml ├── .gitignore ├── .lintr ├── .pre-commit-config.yaml ├── API ├── DESCRIPTION ├── LICENSE ├── LICENSE.md ├── NAMESPACE ├── NEWS.md ├── R ├── analyze.R ├── core.R ├── io.R ├── prepare.R ├── source.R ├── testing.R ├── touchstone-package.R ├── use.R ├── utils.R └── zzz.R ├── README.Rmd ├── README.md ├── _pkgdown.yml ├── actions ├── README.md ├── comment │ └── action.yaml └── receive │ └── action.yaml ├── inst ├── WORDLIST ├── config.json ├── footer.R ├── gitignore ├── header.R ├── script.R ├── touchstone-comment.yaml ├── touchstone-receive.yaml └── workflow-visualization.odt ├── man ├── activate.Rd ├── assert_no_global_installation.Rd ├── benchmark_analyze.Rd ├── benchmark_ls.Rd ├── benchmark_read.Rd ├── benchmark_run.Rd ├── benchmark_run_impl.Rd ├── benchmark_run_iteration.Rd ├── benchmark_verbalize.Rd ├── benchmark_write.Rd ├── branch_get_or_fail.Rd ├── branch_install.Rd ├── branch_install_impl.Rd ├── branches_upsample.Rd ├── cache_up_to_date.Rd ├── figures │ ├── README-example-1.png │ ├── screenshot-pr-comment.png │ └── workflow-visualization.png ├── hash_pkg.Rd ├── install_missing_deps.Rd ├── is_installed.Rd ├── local_clean_touchstone.Rd ├── local_package.Rd ├── local_touchstone_libpath.Rd ├── local_without_touchstone_lib.Rd ├── path_pinned_asset.Rd ├── pin_assets.Rd ├── pr_comment.Rd ├── run_script.Rd ├── touchstone-package.Rd ├── touchstone_managers.Rd ├── touchstone_script.Rd ├── use_touchstone.Rd └── use_touchstone_workflows.Rd ├── tests ├── testthat.R └── testthat │ ├── _snaps │ ├── source.md │ ├── use.md │ ├── use │ │ ├── receive_all.yml │ │ ├── receive_command.yml │ │ ├── receive_default.yml │ │ └── receive_limit.yml │ └── utils.md │ ├── test-analyze.R │ ├── test-core.R │ ├── test-end-to-end.R │ ├── test-io.R │ ├── test-prepare.R │ ├── test-source.R │ ├── test-use.R │ └── test-utils.R ├── touchstone.Rproj └── vignettes ├── .gitignore ├── inference.Rmd └── touchstone.Rmd /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^touchstone\.Rproj$ 2 | ^\.Rproj\.user$ 3 | ^LICENSE\.md$ 4 | ^\.pre-commit-config\.yaml$ 5 | ^\.github$ 6 | ^README\.Rmd$ 7 | ^API$ 8 | ^_pkgdown\.yml$ 9 | ^docs$ 10 | ^pkgdown$ 11 | ^doc$ 12 | ^Meta$ 13 | ^actions$ 14 | ^\.lintr$ 15 | -------------------------------------------------------------------------------- /.github/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "github-actions" # See documentation for possible values 9 | directories: # Location of package manifests 10 | - "/" 11 | - "/actions/comment" 12 | - "/actions/receive" 13 | schedule: 14 | interval: "weekly" 15 | -------------------------------------------------------------------------------- /.github/workflows/check-standard.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | branches: [main, master] 8 | 9 | name: R-CMD-check 10 | 11 | jobs: 12 | R-CMD-check: 13 | runs-on: ${{ matrix.config.os }} 14 | 15 | name: ${{ matrix.config.os }} (${{ matrix.config.r }}) 16 | 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | config: 21 | - {os: macOS-latest, r: 'release'} 22 | - {os: windows-latest, r: 'release'} 23 | - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} 24 | - {os: ubuntu-latest, r: 'release'} 25 | - {os: ubuntu-latest, r: 'oldrel-1'} 26 | 27 | env: 28 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 29 | R_KEEP_PKG_SOURCE: yes 30 | 31 | steps: 32 | - uses: actions/checkout@v4 33 | 34 | - uses: r-lib/actions/setup-pandoc@v2 35 | 36 | - uses: r-lib/actions/setup-r@v2 37 | with: 38 | r-version: ${{ matrix.config.r }} 39 | http-user-agent: ${{ matrix.config.http-user-agent }} 40 | use-public-rspm: true 41 | 42 | # desc gh version fixes a bug in create_package 43 | - uses: r-lib/actions/setup-r-dependencies@v2 44 | with: 45 | extra-packages: | 46 | any::rcmdcheck 47 | github::r-lib/desc@main 48 | needs: check 49 | 50 | - uses: r-lib/actions/check-r-package@v2 51 | 52 | concurrency: 53 | group: ${{ github.workflow }}-${{ github.head_ref }} 54 | cancel-in-progress: true 55 | -------------------------------------------------------------------------------- /.github/workflows/pkgdown.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: main 4 | 5 | name: pkgdown 6 | 7 | jobs: 8 | pkgdown: 9 | runs-on: macOS-latest 10 | env: 11 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - uses: r-lib/actions/setup-r@v2 16 | 17 | - uses: r-lib/actions/setup-pandoc@v2 18 | 19 | - uses: r-lib/actions/setup-r-dependencies@v2 20 | with: 21 | extra-packages: | 22 | any::pkgdown 23 | local::. 24 | 25 | - name: Deploy package 26 | run: | 27 | git config --local user.email "actions@github.com" 28 | git config --local user.name "GitHub Actions" 29 | Rscript -e 'pkgdown::deploy_to_branch(new_process = FALSE)' 30 | 31 | concurrency: 32 | group: ${{ github.workflow }}-${{ github.head_ref }} 33 | cancel-in-progress: true 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | **touchstone/ 5 | docs 6 | inst/doc 7 | /doc/ 8 | /Meta/ 9 | inst/workflow-visualization.pages 10 | -------------------------------------------------------------------------------- /.lintr: -------------------------------------------------------------------------------- 1 | linters: linters_with_defaults( 2 | line_length_linter(120), 3 | brace_linter = NULL, # clashes with {styler} 4 | indentation_linter = NULL, 5 | object_usage_linter = NULL # too many false positives 6 | ) 7 | exclusions: list("inst/script.R","R/zzz.R"=1) 8 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # All available hooks: https://pre-commit.com/hooks.html 2 | # R specific hooks: https://github.com/lorenzwalthert/precommit 3 | ci: 4 | skip: [roxygenize] 5 | 6 | repos: 7 | - repo: https://github.com/lorenzwalthert/precommit 8 | rev: v0.4.3.9008 9 | hooks: 10 | - id: style-files 11 | args: [--style_pkg=styler, --style_fun=tidyverse_style] 12 | - id: lintr 13 | verbose: true 14 | - id: roxygenize 15 | # roxygen requires loading pkg -> add dependencies from DESCRIPTION 16 | additional_dependencies: 17 | - bench 18 | - callr 19 | - cli 20 | - fs 21 | - gert 22 | - magrittr 23 | - purrr 24 | - remotes 25 | - rlang 26 | - tibble 27 | - vctrs 28 | - withr 29 | - r-lib/pkgapi 30 | require_serial: True # https://github.com/pre-commit-ci/runner-image/issues/93#issuecomment-927257158 31 | # codemeta must be above use-tidy-description when both are used 32 | # - id: codemeta-description-updated 33 | - id: use-tidy-description 34 | - id: spell-check 35 | exclude: > 36 | (?x)^( 37 | data/.*| 38 | (.*/|)\.Rprofile| 39 | (.*/|)\.Renviron| 40 | (.*/|)\.gitignore| 41 | (.*/|)NAMESPACE| 42 | (.*/|)WORDLIST| 43 | (.*/|)\.travis.yml| 44 | (.*/|)appveyor.yml| 45 | (.*/|)\.Rbuildignore| 46 | (.*/|)\.pre-commit-.*| 47 | .*\.[rR]| 48 | .*\.Rproj| 49 | .*\.pdf| 50 | .*\.py| 51 | .*\.feather| 52 | .*\.rds| 53 | .*\.Rds| 54 | .*\.sh| 55 | .*\.RData| 56 | .*\.odt| 57 | .*\.png| 58 | \.github/workflows/.*\.yaml| 59 | API| 60 | inst/touchstone\.yaml| 61 | inst/config\.json| 62 | LICENCE\.md| 63 | tests/testthat/_snaps/.*| 64 | API 65 | )$ 66 | - id: readme-rmd-rendered 67 | - id: parsable-R 68 | - id: no-browser-statement 69 | - id: deps-in-desc 70 | exclude: > 71 | (?x)^( 72 | inst/script\.R| 73 | README\.Rmd| 74 | tests/testthat/test-prepare\.R| 75 | inst/script\.R| 76 | vignettes/touchstone\.Rmd 77 | )$ 78 | - repo: https://github.com/pre-commit/pre-commit-hooks 79 | rev: v5.0.0 80 | hooks: 81 | - id: check-added-large-files 82 | args: ['--maxkb=200'] 83 | - id: end-of-file-fixer 84 | exclude: > 85 | (?x)^( 86 | tests/testthat/_snaps/.*| 87 | .*\.Rd 88 | )$ 89 | - repo: local 90 | hooks: 91 | - id: forbid-to-commit 92 | name: Don't commit common R artifacts 93 | entry: Cannot commit .Rhistory, .RData, .Rds or .rds. 94 | language: fail 95 | files: '\.Rhistory|\.RData|\.Rds|\.rds$' 96 | # `exclude: ` to allow committing specific files. 97 | -------------------------------------------------------------------------------- /API: -------------------------------------------------------------------------------- 1 | # API for touchstone package 2 | 3 | ## Exported functions 4 | 5 | activate(head_branch = gert::git_branch(), base_branch = getOption("touchstone.default_base_branch", "main"), n = 1, env = parent.frame()) 6 | benchmark_analyze(branches = c(branch_get_or_fail("GITHUB_BASE_REF"), branch_get_or_fail("GITHUB_HEAD_REF")), names = NULL, ci = 0.95) 7 | benchmark_ls(name) 8 | benchmark_read(name, branch) 9 | benchmark_run(expr_before_benchmark = {}, ..., branches = c(branch_get_or_fail("GITHUB_BASE_REF"), branch_get_or_fail("GITHUB_HEAD_REF")), n = 100, path_pkg = ".") 10 | benchmark_write(benchmark, name, branch, block = NA, iteration = NA, append = TRUE) 11 | branch_get_or_fail(var) 12 | branch_install(branches = c(branch_get_or_fail("GITHUB_BASE_REF"), branch_get_or_fail("GITHUB_HEAD_REF")), path_pkg = ".", install_dependencies = FALSE) 13 | deactivate(env = parent.frame()) 14 | dir_touchstone() 15 | path_pinned_asset(..., branch = branch_get_or_fail("GITHUB_HEAD_REF")) 16 | path_pr_comment() 17 | pin_assets(..., branch = branch_get_or_fail("GITHUB_HEAD_REF"), overwrite = TRUE) 18 | run_script(path = "touchstone/script.R", branch = branch_get_or_fail("GITHUB_HEAD_REF")) 19 | touchstone_clear(all = FALSE) 20 | use_touchstone(overwrite = FALSE, command = NULL, limit_to = c("OWNER", "MEMBER", "COLLABORATOR"), force_upstream = TRUE) 21 | use_touchstone_workflows(overwrite = FALSE, command = NULL, limit_to = c("OWNER", "MEMBER", "COLLABORATOR"), force_upstream = TRUE) 22 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Type: Package 2 | Package: touchstone 3 | Title: Continuous Benchmarking with Statistical Confidence Based on 'Git' Branches 4 | Version: 0.0.0.9002 5 | Authors@R: 6 | c(person(given = "Lorenz", 7 | family = "Walthert", 8 | role = c("aut", "cre"), 9 | email = "lorenz.walthert@icloud.com"), 10 | person(given = "Jacob", 11 | family = "Wujciak-Jens", 12 | role = "aut", 13 | email = "jacob@wujciak.de", 14 | comment = c(ORCID = "0000-0002-7281-3989"))) 15 | Description: A common problem of benchmarking in continuous integration is 16 | that the computational power of the virtual machines that run the job 17 | varies over time. This package allows users to benchmark two branches 18 | of the same repo in a random sequence for better comparison. 19 | License: MIT + file LICENSE 20 | URL: https://github.com/lorenzwalthert/touchstone, 21 | https://lorenzwalthert.github.io/touchstone 22 | BugReports: https://github.com/lorenzwalthert/touchstone/issues 23 | Imports: 24 | bench, 25 | callr, 26 | cli (>= 3.6.1), 27 | fs, 28 | gert, 29 | magrittr, 30 | purrr, 31 | remotes, 32 | rlang, 33 | tibble, 34 | vctrs, 35 | withr 36 | Suggests: 37 | dplyr, 38 | ggplot2, 39 | glue, 40 | knitr, 41 | mockery (>= 0.4.2), 42 | openssl (>= 1.4.1), 43 | rmarkdown, 44 | testthat (>= 3.0.0), 45 | usethis 46 | Remotes: 47 | r-lib/mockery 48 | VignetteBuilder: 49 | knitr 50 | Config/testthat/edition: 3 51 | Config/testthat/parallel: true 52 | Encoding: UTF-8 53 | LazyData: true 54 | Roxygen: list(markdown = TRUE, roclets = c("rd", "namespace", "collate", 55 | if (rlang::is_installed("pkgapi")) "pkgapi::api_roclet" else { 56 | warning("Please install r-lib/pkgapi to make sure the file API is kept 57 | up to date"); NULL})) 58 | RoxygenNote: 7.2.1 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | YEAR: 2020 2 | COPYRIGHT HOLDER: Lorenz Walthert 3 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # MIT License 2 | 3 | Copyright (c) 2020 Lorenz Walthert 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | export(activate) 4 | export(benchmark_analyze) 5 | export(benchmark_ls) 6 | export(benchmark_read) 7 | export(benchmark_run) 8 | export(benchmark_write) 9 | export(branch_get_or_fail) 10 | export(branch_install) 11 | export(deactivate) 12 | export(dir_touchstone) 13 | export(path_pinned_asset) 14 | export(path_pr_comment) 15 | export(pin_assets) 16 | export(run_script) 17 | export(touchstone_clear) 18 | export(use_touchstone) 19 | export(use_touchstone_workflows) 20 | importFrom(magrittr,"%>%") 21 | importFrom(rlang,"%|%") 22 | importFrom(rlang,.data) 23 | importFrom(tibble,lst) 24 | importFrom(tibble,tibble) 25 | -------------------------------------------------------------------------------- /NEWS.md: -------------------------------------------------------------------------------- 1 | # touchstone 0.1.0-rc 2 | 3 | This is the initial CRAN release of {touchstone}. Please see `README.md` and 4 | `vignette("touchstone", package = "touchstone")` on how to get started. 5 | 6 | **Breaking Changes to dev version** 7 | 8 | * Complete API overhaul with many breaking changes (#83). 9 | * `with_touchstone_lib()` was renamed to `run_script()`. 10 | * `benchmark_run_ref()` does not accept a character input for code expressions 11 | anymore (#62). 12 | -------------------------------------------------------------------------------- /R/analyze.R: -------------------------------------------------------------------------------- 1 | #' Turn raw benchmark results into text and figures 2 | #' 3 | #' @details 4 | #' Creates two side effects: 5 | #' 6 | #' * Density plots for each element in `branches` are written to 7 | #' `touchstone/plots`. 8 | #' * A text explaining the speed diff is written to 9 | #' `touchstone/pr-comment/info.txt` for every registered benchmarking 10 | #' expression. See `vignette("inference", package = "touchstone")` for 11 | #' details. 12 | #' @param branches The names of the branches for which analysis should be 13 | #' created. 14 | #' @param names The names of the benchmarks to analyze. If `NULL`, all 15 | #' benchmarks with the `branches` are taken. 16 | #' @param ci The confidence level, defaults to 95%. 17 | #' @details Requires [dplyr::dplyr], [ggplot2::ggplot2] and [glue::glue]. 18 | #' @return 19 | #' A character vector that summarizes the benchmarking results. 20 | #' @export 21 | benchmark_analyze <- function(branches = c( 22 | branch_get_or_fail("GITHUB_BASE_REF"), 23 | branch_get_or_fail("GITHUB_HEAD_REF") 24 | ), 25 | names = NULL, 26 | ci = 0.95) { 27 | suggested_pkgs <- c("dplyr", "ggplot2", "glue") 28 | suggests_available <- purrr::map_lgl( 29 | suggested_pkgs, 30 | requireNamespace, 31 | quietly = TRUE 32 | ) 33 | 34 | if (!all(suggests_available)) { 35 | missing_pkgs <- suggested_pkgs[!suggests_available] 36 | n_pkgs <- length(missing_pkgs) 37 | pkgs_str <- paste0('"', missing_pkgs, '"', collapse = ",") 38 | cli::cli_abort(c( 39 | "Analysing the benchmarks requires {n_pkgs} additional package{?s}!", 40 | "i" = "To install use {.code install.packages(c({pkgs_str}))}" 41 | )) 42 | } 43 | 44 | if (length(branches) != 2) { 45 | cli::cli_abort("There must be exactly two branches to comare.") 46 | } 47 | path_info <- path_pr_comment() 48 | default_header <- paste0( 49 | "This is how benchmark results would change (along with a ", 100 * ci, 50 | "% confidence interval in relative change) if ", 51 | system2("git", c("rev-parse", "HEAD"), stdout = TRUE), 52 | " is merged into ", branches[1], ":", "\n" 53 | ) 54 | 55 | get_comment_text("header", default_header) %>% 56 | writeLines(path_info) 57 | 58 | if (is.null(names)) { 59 | # only select names that occur exactly twice 60 | names <- benchmark_ls() %>% 61 | dplyr::filter(.data$branch %in% !!branches) %>% 62 | dplyr::group_by(.data$name) %>% 63 | dplyr::count() 64 | 65 | filtered_names <- dplyr::filter(names, .data$n == 2) 66 | if (!identical(names, filtered_names)) { 67 | cli::cli_warn(c( 68 | "All benchmarks to analyse must have the two branches {.val {branches[[1]]}} and {.val {branches[[2]]}}", 69 | "!" = "Ignoring all benchmarks that don't have exactly those two branches.", 70 | "i" = "To avoid this warning, inspect the existing benchmarks with {.fun touchstone::benchmark_ls}" 71 | )) 72 | names <- filtered_names 73 | } 74 | } 75 | 76 | out <- purrr::walk( 77 | names$name, 78 | benchmark_analyze_impl, 79 | branches = branches, ci = ci 80 | ) 81 | default_footer <- paste( 82 | "\nFurther explanation regarding interpretation and methodology can be found", 83 | "in the [documentation](https://lorenzwalthert.github.io/touchstone/articles/inference.html)." 84 | ) 85 | text <- get_comment_text("footer", default_footer) 86 | cat(text, fill = TRUE, file = path_info, append = TRUE) 87 | 88 | readLines(path_info) 89 | } 90 | 91 | #' @importFrom rlang .data 92 | benchmark_analyze_impl <- function(benchmark, branches = c( 93 | branch_get_or_fail("GITHUB_BASE_REF"), 94 | branch_get_or_fail("GITHUB_HEAD_REF") 95 | ), 96 | ci = 0.95) { 97 | timings <- benchmark_read(benchmark, branches) 98 | benchmark_plot(benchmark, timings) 99 | benchmark_verbalize(benchmark, timings = timings, branches = branches, ci = ci) 100 | } 101 | 102 | #' Create nice text from benchmarks 103 | #' 104 | #' `branches` must be passed because the order is relevant. 105 | #' @inheritParams benchmark_plot 106 | #' @keywords internal 107 | benchmark_verbalize <- function(benchmark, timings, branches, ci) { 108 | tbl <- timings %>% 109 | dplyr::group_by(.data$branch) %>% 110 | dplyr::summarise( 111 | mean = bench::as_bench_time(mean(.data$elapsed)), 112 | sd = stats::sd(.data$elapsed) 113 | ) %>% 114 | dplyr::inner_join(tibble::tibble(branch = branches), ., by = "branch") 115 | 116 | if (nrow(tbl) > 2) { 117 | cli::cli_abort("Benchmarks with more than two {.val branches} cannot be verbalized.") 118 | } 119 | confint <- confint_relative_get(timings, branches, as.numeric(tbl$mean[1]), ci = ci) 120 | 121 | text <- glue::glue( 122 | "* {confint$emoji}{benchmark}: {tbl$mean[1]} -> {tbl$mean[2]} {confint$string}" 123 | ) 124 | cat( 125 | text, 126 | fill = TRUE, file = path_pr_comment(), 127 | append = TRUE 128 | ) 129 | text 130 | } 131 | 132 | set_sign <- function(x) { 133 | purrr::map_chr(x, ~ paste0(ifelse(.x > 0, "+", ""), .x)) 134 | } 135 | 136 | confint_relative_get <- function(timings, branches, reference, ci) { 137 | no_change <- ":heavy_check_mark:" 138 | slower <- ":exclamation::snail:" 139 | faster <- ":rocket:" 140 | 141 | timings_with_factors <- timings %>% 142 | dplyr::mutate( 143 | block = factor(.data$block), branch = factor(.data$branch, levels = branches) 144 | ) 145 | stopifnot(inherits(timings_with_factors$branch, "factor")) 146 | fit <- stats::aov(elapsed ~ branch, data = timings_with_factors) 147 | var <- paste0("branch", branches[2]) 148 | confint <- confint(fit, var, level = ci) 149 | confint <- round(100 * confint / reference, 2) 150 | emoji <- if (all(confint < 0)) { 151 | faster 152 | } else if (all(confint > 0)) { 153 | slower 154 | } else { 155 | no_change 156 | } 157 | list( 158 | string = paste0( 159 | "[", 160 | paste0(set_sign(confint), collapse = "%, "), 161 | "%]" 162 | ), 163 | emoji = emoji 164 | ) 165 | } 166 | 167 | 168 | #' @param timing a benchmark read with [benchmark_read()], column `name` must 169 | #' only contain one unique value. 170 | #' @keywords internal 171 | benchmark_plot <- function(benchmark, timings) { 172 | timings %>% 173 | ggplot2::ggplot(ggplot2::aes(x = .data$elapsed, color = .data$branch)) + 174 | ggplot2::geom_density() 175 | fs::path(dir_touchstone(), "plots", benchmark) %>% 176 | fs::path_ext_set("png") %>% 177 | ggplot2::ggsave() 178 | } 179 | 180 | 181 | #' Modifying the PR Comment 182 | #' 183 | #' The files `touchstone/header.R` and `touchstone/footer.R` allow you to modify 184 | #' the PR comment. The files will be evaluated in the context of 185 | #' [benchmark_analyze()] and should return one string containing the text. 186 | #' You can use github markdown e.g. emojis like :tada: in the string. 187 | #' 188 | #' @section Header: 189 | #' Available variables for glue substitution: 190 | #' * ci: confidence interval 191 | #' * branches: BASE and HEAD branches benchmarked against each other. 192 | #' 193 | #' @section Footer: 194 | #' There are no special variables available in the footer. 195 | #' You can access the benchmark results via [path_pr_comment()]. 196 | #' 197 | #' @name pr_comment 198 | #' @seealso [base::eval()] [base::parse()] 199 | NULL 200 | 201 | get_comment_text <- function(part = c("footer", "header"), default, env = parent.frame()) { 202 | part <- match.arg(part) 203 | file <- glue::glue("{part}.R") 204 | path <- fs::path(dir_touchstone(), file) 205 | 206 | if (!fs::file_exists(path)) { 207 | cli::cli_alert_info("No comment {part} found. Using default.") 208 | text <- default 209 | } else { 210 | text <- eval(parse(path), envir = env) 211 | if (!is.character(text)) { 212 | cli::cli_warn( 213 | c("Parsed comment {part} is not a valid string. Using default.", 214 | "i" = "See {.code ?touchstone::pr_comment} for more information." 215 | ) 216 | ) 217 | text <- default 218 | } 219 | } 220 | 221 | text 222 | } 223 | -------------------------------------------------------------------------------- /R/core.R: -------------------------------------------------------------------------------- 1 | #' Run a benchmark iteration 2 | #' @param expr_before_benchmark Expression to run before 3 | #' the benchmark is ran, will be captured with [rlang::enexpr()]. So you can 4 | #' use quasiquotation. 5 | #' @param n Number of iterations to run a benchmark within an iteration. 6 | #' @param dots list of quoted expressions (length 1). 7 | #' @inheritParams benchmark_write 8 | #' @importFrom tibble lst tibble 9 | #' @keywords internal 10 | benchmark_run_iteration <- function(expr_before_benchmark, 11 | dots, 12 | branch, 13 | block, 14 | n = getOption("touchstone.n_iterations", 1)) { 15 | if (rlang::is_missing(expr_before_benchmark)) { 16 | expr_before_benchmark <- rlang::expr({}) 17 | } 18 | 19 | args <- rlang::list2( 20 | expr_before_benchmark = expr_before_benchmark, 21 | dots = dots, 22 | branch = branch, 23 | block = block, 24 | asset_dirs = options() %>% 25 | names() %>% 26 | grep("touchstone.dir_assets_", .) %>% 27 | options()[.] 28 | ) 29 | 30 | for (iteration in seq_len(n)) { 31 | # iterations 32 | callr::r( 33 | function(expr_before_benchmark, dots, branch, block, iteration, asset_dirs) { 34 | withr::local_namespace("touchstone") 35 | withr::local_options(asset_dirs) 36 | eval(expr_before_benchmark) 37 | benchmark <- bench::mark(eval(dots[[1]]), check = FALSE, memory = FALSE, iterations = 1) 38 | benchmark_write(benchmark, names(dots), branch = branch, block = block, iteration = iteration) 39 | }, 40 | args = append(args, lst(iteration)), 41 | libpath = c(libpath_touchstone(branch), .libPaths()) 42 | ) 43 | } 44 | cli::cli_alert_success("Ran {n} iteration{?s} of branch {.val {branch}}.") 45 | benchmark_read(names(dots), branch) 46 | } 47 | 48 | #' Run a benchmark for git branches 49 | #' 50 | #' @param ... Named expression of length one with code to benchmark, 51 | #' will be captured with [rlang::enexprs()]. So you can use quasiquotation. 52 | #' @param branches Character vector with branch names to benchmark. The package 53 | #' must be built for each benchmarked branch beforehand with [branch_install()]. 54 | #' The base branch is the target branch of the pull request in a workflow run, 55 | #' the head branch is the source branch of the pull request in a workflow run. 56 | #' @param n Number of times benchmarks should be run for each `branch`. The more 57 | #' iterations you run, the more narrow your confidence interval will be and 58 | #' the smaller the differences you will detect. See also 59 | #' `vignette("inference")`. To simplify interactive experimentation with 60 | #' `benchmark_run()`, `n` will be overridden in interactive usage after the 61 | #' user calls `activate(..., n = 1)`. 62 | #' @param path_pkg The path to the package to benchmark. Will be used to 63 | #' temporarily checkout the branch during benchmarking. 64 | #' @inheritParams branch_install 65 | #' @inheritParams benchmark_run_impl 66 | #' @details 67 | #' Runs the following loop `n` times: 68 | #' * removes all touchstone libraries from the library path, adding the one 69 | #' corresponding to `branch`. 70 | #' * runs setup code `exp_before_branch`. 71 | #' * benchmarks `expr_to_benchmark` and writes them to disk. 72 | #' 73 | #' @return 74 | #' All timings in a tibble. 75 | #' @section Caution: 76 | #' This function will perform various git operations that affect the state of 77 | #' the directory it is ran in, in particular different branches will be checked 78 | #' out. Ensure a clean git working directory before invocation. 79 | #' @export 80 | benchmark_run <- function(expr_before_benchmark = 81 | {}, 82 | ..., 83 | branches = c( 84 | branch_get_or_fail("GITHUB_BASE_REF"), 85 | branch_get_or_fail("GITHUB_HEAD_REF") 86 | ), 87 | n = 100, 88 | path_pkg = ".") { 89 | force(branches) 90 | expr_before_benchmark <- rlang::enexpr(expr_before_benchmark) 91 | dots <- rlang::enexprs(...) 92 | 93 | if (length(dots) > 1) { 94 | cli::cli_abort("Expression to benchmark cannot have length greater than one.") 95 | } 96 | if (rlang::is_string(dots[[1]]) || 97 | rlang::is_string(expr_before_benchmark)) { 98 | abort_string() 99 | } 100 | 101 | if (rlang::is_interactive()) { 102 | new_n <- getOption("touchstone.n_runs") 103 | if (!is.null(new_n) && new_n != n) { 104 | cli::cli_alert_info(paste0( 105 | "{.fun activate} has overriden n = {n} with {new_n} since ", 106 | "{.fun rlang::is_interactive } is {.code TRUE}." 107 | )) 108 | n <- new_n 109 | } 110 | } 111 | 112 | # touchstone libraries must be removed from the path temporarily 113 | # and the one to benchmark will be added in benchmark_run_impl() 114 | local_without_touchstone_lib() 115 | gh_cat(glue::glue("::group::Benchmark: {names(dots)}\n\n")) 116 | withr::defer(gh_cat("::endgroup::\n")) 117 | branches <- branches_upsample(branches, n = n) 118 | cli::cli_alert_info("Running {2*n} iterations of benchmark: {names(dots)}.") 119 | out_list <- purrr::pmap(branches, benchmark_run_impl, 120 | expr_before_benchmark = expr_before_benchmark, 121 | dots = dots, 122 | path_pkg = path_pkg 123 | ) 124 | vctrs::vec_rbind(!!!out_list) 125 | } 126 | 127 | #' Checkout a branch from a repo and run an iteration 128 | #' 129 | #' @param path_pkg The path to the root of the package you want to benchmark. 130 | #' @inheritParams benchmark_run_iteration 131 | #' @keywords internal 132 | benchmark_run_impl <- function(branch, 133 | block, 134 | expr_before_benchmark, 135 | dots, 136 | path_pkg) { 137 | local_git_checkout(branch, path_pkg) 138 | benchmark_run_iteration( 139 | expr_before_benchmark = expr_before_benchmark, 140 | dots = dots, 141 | branch = branch, 142 | block = block 143 | ) 144 | } 145 | -------------------------------------------------------------------------------- /R/io.R: -------------------------------------------------------------------------------- 1 | #' Write a benchmark 2 | #' 3 | #' @param benchmark The result of [bench::mark()], with `iterations = 1`. 4 | #' @param name The name of the benchmark. 5 | #' @param append Whether to append the result to the file or not. 6 | #' @param branch A character vector of length one to indicate the git branch (i.e. 7 | #' commit, tag, branch etc) of the benchmarking. 8 | #' @param block All branches that appear once in a block. 9 | #' @param iteration An integer indicating to which iteration the benchmark 10 | #' refers to. Multiple iterations within a block always benchmark the same 11 | #' `branch`. 12 | #' @return 13 | #' Character vector of length one with path to the record written (invisibly). 14 | #' @export 15 | benchmark_write <- function(benchmark, name, branch, block = NA, iteration = NA, append = TRUE) { 16 | if (benchmark$n_itr > 1) { 17 | cli::cli_abort("This package only supports benchmarks with {.code bench::mark(..., iterations = 1)}.") 18 | } 19 | path <- path_record(name, branch) 20 | init_touchstone() 21 | ensure_dir(fs::path_dir(path)) 22 | tibble( 23 | elapsed = as.numeric(benchmark$median), 24 | iteration = iteration, 25 | branch = enc2utf8(branch), 26 | block = block, 27 | name = name 28 | ) %>% 29 | benchmark_write_impl(path = path, append = append) 30 | } 31 | 32 | init_touchstone <- function() { 33 | ensure_dir(dir_touchstone(), "plots") 34 | ensure_dir(dir_touchstone(), "pr-comment") 35 | ensure_dir(dir_touchstone(), "records") 36 | } 37 | 38 | benchmark_write_impl <- function(benchmark, path, append) { 39 | file_exists <- fs::file_exists(path) 40 | suppressWarnings( 41 | utils::write.table( 42 | benchmark, path, 43 | append = append, row.names = FALSE, 44 | fileEncoding = "UTF-8", col.names = !file_exists 45 | ) 46 | ) 47 | invisible(path) 48 | } 49 | 50 | #' Read benchmarks 51 | #' @inheritParams benchmark_write 52 | #' @return 53 | #' A tibble with the benchmarks. 54 | #' @export 55 | benchmark_read <- function(name, branch) { 56 | path_outputs <- path_record(name, branch) 57 | out <- purrr::map( 58 | path_outputs, 59 | benchmark_read_impl 60 | ) 61 | vctrs::vec_rbind(!!!out) 62 | } 63 | 64 | 65 | new_benchmark_ls_tibble <- function(name = character(), branch = character()) { 66 | tibble::tibble(name, branch) 67 | } 68 | 69 | #' List which benchmarks were recorded 70 | #' 71 | #' @inheritParams benchmark_write 72 | #' @return 73 | #' A tibble with name and branches of the existing benchmarks. 74 | #' @export 75 | benchmark_ls <- function(name = "") { 76 | path_record <- path_record() 77 | if (!fs::dir_exists(path_record())) { 78 | return(new_benchmark_ls_tibble()) 79 | } 80 | path_names <- fs::dir_ls(path_record, type = "directory") 81 | if (length(path_names) < 1) { 82 | return(new_benchmark_ls_tibble()) 83 | } 84 | all_names <- fs::path_file(path_names) 85 | path <- path_record(name = all_names) 86 | dirs <- fs::dir_ls(path, type = "file") 87 | new_benchmark_ls_tibble( 88 | name = fs::path_file(fs::path_dir(dirs)), 89 | branch = fs::path_file(dirs) 90 | ) 91 | } 92 | 93 | path_record <- function(name = "", branch = "") { 94 | as.character(fs::path(dir_touchstone(), "records", name, branch)) 95 | } 96 | 97 | benchmark_read_impl <- function(path) { 98 | utils::read.table(path, header = TRUE, colClasses = schema_disk()) %>% 99 | tibble::as_tibble() 100 | } 101 | -------------------------------------------------------------------------------- /R/prepare.R: -------------------------------------------------------------------------------- 1 | #' Checks out a source branch and install the package 2 | #' 3 | #' @param path_pkg The path to the repository to install. 4 | #' @param branch The name of the branch which should be installed. 5 | #' @param install_dependencies Passed to [remotes::install_local()]. Set to 6 | #' `FALSE` can help when ran locally without internet connection. 7 | #' @return 8 | #' A character vector with library paths. 9 | #' @keywords internal 10 | branch_install_impl <- function(branch = "main", 11 | path_pkg = ".", 12 | install_dependencies = FALSE) { 13 | local_git_checkout(branch, path_pkg) 14 | if (getOption("touchstone.skip_install", FALSE)) { 15 | cli::cli_alert_info( 16 | "R option {.envvar touchstone.skip_install} is set, skipping installation." 17 | ) 18 | NULL 19 | } else { 20 | local_touchstone_libpath(branch) 21 | libpath <- .libPaths() 22 | install_local <- purrr::partial(remotes::install_local, path_pkg, 23 | upgrade = "never", 24 | dependencies = install_dependencies, 25 | force = !cache_up_to_date(branch, path_pkg) 26 | ) 27 | withr::local_options(warn = 2) 28 | rlang::try_fetch( 29 | { 30 | install_missing_deps(path_pkg = path_pkg, quiet = TRUE) 31 | install_local(quiet = TRUE) 32 | }, 33 | error = function(e) { 34 | install_missing_deps(path_pkg = path_pkg, quiet = FALSE) 35 | install_local(quiet = FALSE) 36 | } 37 | ) 38 | cache_update(branch, path_pkg) 39 | cli::cli_alert_success("Installed branch {.val {branch}} into {.path {libpath[1]}}.") 40 | libpath 41 | } 42 | } 43 | 44 | #' Install branches 45 | #' 46 | #' Installs each `branch` in a separate library for isolation. 47 | #' @param branches The names of the branches in a character vector. 48 | #' @param install_dependencies Passed to [remotes::install_local()]. 49 | #' @inheritParams branch_install_impl 50 | #' @return 51 | #' The global and touchstone library paths in a character vector (invisibly). 52 | #' @export 53 | branch_install <- function(branches = c( 54 | branch_get_or_fail("GITHUB_BASE_REF"), 55 | branch_get_or_fail("GITHUB_HEAD_REF") 56 | ), 57 | path_pkg = ".", 58 | install_dependencies = FALSE) { 59 | force(branches) 60 | assert_no_global_installation(path_pkg) 61 | gh_cat("::group::Installing branches\n\n") 62 | withr::defer(gh_cat("::endgroup::\n")) 63 | cli::cli_alert_info("Start installing branches into separate libraries.") 64 | libpaths <- purrr::map(branches, branch_install_impl, 65 | path_pkg = path_pkg, 66 | install_dependencies = install_dependencies 67 | ) %>% 68 | purrr::flatten_chr() %>% 69 | unique() %>% 70 | fs::path_abs() %>% 71 | as.character() %>% 72 | sort() 73 | assert_no_global_installation(path_pkg) 74 | cli::cli_alert_success("Completed installations.") 75 | invisible(libpaths) 76 | } 77 | 78 | 79 | 80 | libpath_touchstone <- function(branch) { 81 | fs::path(dir_touchstone(), "lib", branch) 82 | } 83 | 84 | #' When did the package sources change last? 85 | #' @inheritParams branch_install 86 | #' @keywords internal 87 | hash_pkg <- function(path_pkg) { 88 | withr::local_dir(path_pkg) 89 | list( 90 | tools::md5sum(c( 91 | if (fs::dir_exists("R")) fs::dir_ls("R"), 92 | if (fs::file_exists("DESCRIPTION")) "DESCRIPTION", 93 | if (fs::dir_exists("scr")) fs::dir_info("scr") 94 | )) 95 | ) 96 | } 97 | 98 | #' Cache package sources within a session 99 | #' 100 | #' This is required to make sure [remotes::install_local()] installs again 101 | #' when source code changed. 102 | #' @inheritParams branch_install 103 | #' @keywords internal 104 | cache_up_to_date <- function(branch, path_pkg) { 105 | md5_hashes <- hash_pkg(path_pkg) 106 | cache <- cache_get() 107 | identical(md5_hashes, cache$md5_hashes[cache$branch == branch & cache$path_pkg == path_pkg]) 108 | } 109 | 110 | #' @rdname cache_up_to_date 111 | #' @keywords internal 112 | cache_update <- function(branch, path_pkg) { 113 | md5_hashes <- hash_pkg(path_pkg) 114 | cache <- cache_get() 115 | stopifnot(sum(cache$branch[cache$path_pkg == path_pkg] == branch) <= 1) 116 | cache <- cache[(!(cache$branch == branch) & (cache$path_pkg == path_pkg)), ] 117 | cache <- vctrs::vec_rbind( 118 | cache, tibble::tibble(branch, md5_hashes, path_pkg) 119 | ) 120 | options("touchstone.hash_source_package" = cache) 121 | } 122 | 123 | 124 | #' @rdname cache_up_to_date 125 | #' @keywords internal 126 | cache_get <- function() { 127 | getOption("touchstone.hash_source_package") 128 | } 129 | 130 | #' Install missing BASE dependencies 131 | #' 132 | #' If the HEAD branch removes dependencies (compared to BASE), installing the 133 | #' package from the BASE branch will fail due to missing dependencies, as 134 | #' dependencies were installed in the GitHub Action based on the `DESCRIPTION` 135 | #' of the HEAD branch. The simplest way to ensure all required dependencies are 136 | #' present (including specification of remotes) is by simply installing them 137 | #' into the respective {touchstone} library. Prepend the local touchstone 138 | #' library to the library path with [local_touchstone_libpath()]. 139 | #' @keywords internal 140 | install_missing_deps <- function(path_pkg, quiet = FALSE) { 141 | remotes::install_deps(pkgdir = path_pkg, upgrade = "always", quiet = quiet) 142 | } 143 | -------------------------------------------------------------------------------- /R/source.R: -------------------------------------------------------------------------------- 1 | #' Sources a script 2 | #' 3 | #' Basically [base::source()], but prepending the library path with a 4 | #' touchstone library and running the script in a temp directory to avoid 5 | #' git operations like checking out different branches to interfere with the 6 | #' script execution (as running the script changes itself through git checkout). 7 | #' 8 | #' @param path The script to run. It must fulfill the requirements of a 9 | #' [touchstone_script]. 10 | #' @param branch The branch that corresponds to the library that should be prepended 11 | #' to the library path when the script at `path` is executed, see 'Why this 12 | #' function?' below. 13 | #' 14 | #' @section How to run this interactively?: 15 | #' You can use [activate()] to setup the environment to interactively run your 16 | #' script, as there are some adjustments needed to mirror the Github Action 17 | #' environment. 18 | #' In a GitHub Action workflow, the environment variables `GITHUB_BASE_REF` and 19 | #' `GITHUB_HEAD_REF` denote the target and source branch of the pull request - 20 | #' and these are default arguments in [benchmark_run()] (and other functions 21 | #' you probably want to call in your benchmarking script) to determinate the 22 | #' branches to use. 23 | #' 24 | #' @section Why this function?: 25 | #' For isolation, \{touchstone\} does not allow the benchmarked package to be 26 | #' installed in the global package library, but only in touchstone libraries, as 27 | #' asserted with [assert_no_global_installation()]. However, this also implies 28 | #' that the package is not available in the touchstone script outside of 29 | #' benchmark runs (i.e. outside of [benchmark_run()]. We sometimes still 30 | #' want to call that package to prepare a benchmarking run though. To 31 | #' allow this, we prepend a touchstone library location that 32 | #' contains the installed benchmarked package for set-up tasks, and temporarily 33 | #' remove it during benchmarking with [benchmark_run()] so only one 34 | #' touchstone library is on the library path at any time. 35 | #' 36 | #' @return 37 | #' The same as [base::source()], which inherits from [base::withVisible()], i.e. 38 | #' a list with `value` and `visible` (invisibly). 39 | #' @export 40 | #' @examples 41 | #' \dontrun{ 42 | #' # assuming you want to compare the branch main with the branch devel 43 | #' if (rlang::is_installed("withr")) { 44 | #' withr::with_envvar( 45 | #' c("GITHUB_BASE_REF" = "main", "GITHUB_HEAD_REF" = "devel"), 46 | #' run_script("touchstone/script.R") 47 | #' ) 48 | #' } 49 | #' } 50 | run_script <- function(path = "touchstone/script.R", 51 | branch = branch_get_or_fail("GITHUB_HEAD_REF")) { 52 | force(branch) 53 | rlang::with_interactive( 54 | activate(branch, branch_get_or_fail("GITHUB_BASE_REF")), TRUE 55 | ) 56 | 57 | temp_file <- fs::file_temp() 58 | fs::file_copy(path, temp_file) 59 | 60 | cli::cli_alert_success(paste0( 61 | "Copied touchstone script to tempdir to prevent branch checkouts to effect", 62 | " the script." 63 | )) 64 | 65 | source(temp_file, max.deparse.length = Inf, local = TRUE) 66 | } 67 | 68 | #' Activate touchstone environment 69 | #' 70 | #' This sets environment variables, R options and library paths to work 71 | #' interactively on the [touchstone_script]. 72 | #' @param head_branch Git branch to be used as the `GITHUB_HEAD_REF` branch 73 | #' (i.e. the branch with new changes) when running benchmarks. Defaults to the 74 | #' current branch. 75 | #' @param base_branch Git branch for the `GITHUB_BASE_REF` (i.e. the branch you 76 | #' want to merge your changes into) when running benchmarks. Defaults to 'main' 77 | #' if the option `touchstone.default_base_branch`is not set. 78 | #' @param n Number of times benchmarks should be run for each `branch`. Will 79 | #' override `n` argument in all interactive calls to [benchmark_run()]. 80 | #' @param env In which environment the temporary changes should be made. 81 | #' For use within functions. 82 | #' @examples 83 | #' \dontrun{ 84 | #' activate() 85 | #' # You can now test parts of your touchstone script, e.g. touchstone/script.R 86 | #' deactivate() 87 | #' } 88 | #' @export 89 | activate <- function(head_branch = gert::git_branch(), 90 | base_branch = getOption( 91 | "touchstone.default_base_branch", 92 | "main" 93 | ), 94 | n = 1, 95 | env = parent.frame()) { 96 | if (!rlang::is_interactive()) { 97 | if (envvar_true("GITHUB_ACTIONS")) { 98 | cat(paste0( 99 | "::warning ::activate() is meant for interactive use ", 100 | "only, make sure script.R works as intended!" 101 | )) 102 | } else { 103 | cli::cli_warn(paste0( 104 | "{.fun activate} is meant for interactive use", 105 | "only, make sure {.file script.R} works as intended!" 106 | )) 107 | } 108 | } 109 | 110 | 111 | suppressMessages({ 112 | withr::local_envvar( 113 | GITHUB_BASE_REF = base_branch, 114 | GITHUB_HEAD_REF = head_branch, 115 | .local_envir = env 116 | ) 117 | withr::local_options(touchstone.n_runs = n, .local_envir = env) 118 | 119 | local_touchstone_libpath(head_branch, env = env) 120 | local_asset_dir(base_branch, head_branch, env = env) 121 | }) 122 | 123 | if (identical(env, .GlobalEnv)) { 124 | cli::cli_alert_success( 125 | "Environment ready to interactively execute your touchstone script." 126 | ) 127 | cli::cli_alert_info( 128 | "Use {.fun touchstone::deactivate} to restore original environment." 129 | ) 130 | } 131 | } 132 | 133 | #' Set Library Path 134 | #' 135 | #' Temporarily add a touchstone library to the path, so it can be found by 136 | #' [.libPaths()] and friends. Can be used in [touchstone_script] 137 | #' to prepare benchmarks etc. If there are touchstone libraries on the path 138 | #' when this function is called, they will be removed. 139 | #' @param branch Git branch to use, e.g. HEAD or BASE branch. 140 | #' @param env Environment in which the change should be applied. 141 | #' @seealso [run_script()] 142 | #' @keywords internal 143 | local_touchstone_libpath <- function(branch, env = parent.frame()) { 144 | lib <- libpath_touchstone(branch) 145 | fs::dir_create(lib) 146 | current <- fs::path_real(.libPaths()) 147 | 148 | current_is_touchstone <- purrr::map_lgl(current, 149 | fs::path_has_parent, 150 | parent = fs::path_real(dir_touchstone()) 151 | ) 152 | current <- current[!current_is_touchstone] 153 | withr::local_libpaths( 154 | c(lib, current), 155 | action = "replace", 156 | .local_envir = env 157 | ) 158 | } 159 | 160 | #' @describeIn activate Restore the original environment state. 161 | #' @export 162 | deactivate <- function(env = parent.frame()) { 163 | withr::deferred_clear(envir = env) 164 | cli::cli_alert_success("Original environment restored!") 165 | } 166 | 167 | 168 | #' The script for benchmarking 169 | #' 170 | #' The script that contains the code which executes the benchmark. It is 171 | #' typically called with [run_script()]. 172 | #' 173 | #' @name touchstone_script 174 | #' @section Requirements: 175 | #' 176 | #' A touchstone script must: 177 | #' 178 | #' * install all versions of the benchmarked repository with [branch_install()]. 179 | #' * create benchmarks with one or more calls to [benchmark_run()]. 180 | #' * produce the artifacts required in the GitHub workflow with 181 | #' [benchmark_analyze()]. 182 | NULL 183 | -------------------------------------------------------------------------------- /R/testing.R: -------------------------------------------------------------------------------- 1 | #' Clean up 2 | #' 3 | #' Deletes [dir_touchstone()] when the local frame is destroyed. 4 | #' @inheritParams withr::defer 5 | #' @family testers 6 | #' @keywords internal 7 | local_clean_touchstone <- function(envir = parent.frame()) { 8 | withr::defer(touchstone_clear(all = TRUE), envir = envir) 9 | } 10 | 11 | path_temp_pkg <- function(name) { 12 | fs::path_temp(openssl::md5(as.character(Sys.time())), name) 13 | } 14 | 15 | 16 | #' Create a test package 17 | #' 18 | #' Creates a package in a temporary directory and sets the working directory 19 | #' until the local frame is destroyed. 20 | #' 21 | #' This is primarily for testing. 22 | #' @param pkg_name The name of the temporary package. 23 | #' @param setwd Whether or not the working directory should be temporarily 24 | #' set to the package root. 25 | #' @param branches Branches to be created. 26 | #' @param r_sample Character with code to write to `R/sampleR.`. This is helpful 27 | #' to validate if the installed package corresponds to source branch for 28 | #' testing. If `NULL`, nothing is written. 29 | #' @inheritParams withr::defer 30 | #' @family testers 31 | #' @keywords internal 32 | local_package <- function(pkg_name = fs::path_file(fs::file_temp("pkg")), 33 | branches = c("main", "devel"), 34 | r_sample = NULL, 35 | setwd = TRUE, 36 | envir = parent.frame()) { 37 | path <- fs::path(fs::file_temp(""), pkg_name) 38 | fs::dir_create(path) 39 | withr::local_options( 40 | usethis.quiet = TRUE, 41 | touchstone.n_iterations = 2, 42 | .local_envir = envir, 43 | touchstone.hash_source_package = tibble::tibble( 44 | branch = character(), md5_hashes = list(), path_pkg = character() 45 | ) 46 | ) 47 | usethis::create_package(path, open = FALSE) 48 | withr::local_dir(path, .local_envir = if (setwd) envir else rlang::current_env()) 49 | gert::git_init() 50 | gert::git_config_set("user.name", "GitHub Actions") 51 | gert::git_config_set("user.email", "actions@github.com") 52 | gert::git_add("DESCRIPTION") 53 | writeLines(if (is.null(r_sample)) "" else r_sample, fs::path("R", "sample.R")) 54 | gert::git_add("R/") 55 | gert::git_commit("[init]") 56 | branches <- gert::git_branch_list() %>% 57 | dplyr::pull(.data$name) %>% 58 | dplyr::setdiff(branches, .) 59 | purrr::walk(branches, gert::git_branch_create) 60 | withr::defer(unlink(path), envir = envir) 61 | install_check <- is_installed(path) 62 | if (install_check$installed) { 63 | withr::defer(utils::remove.packages(install_check$name), envir = envir) 64 | } 65 | path 66 | } 67 | -------------------------------------------------------------------------------- /R/touchstone-package.R: -------------------------------------------------------------------------------- 1 | #' @keywords internal 2 | #' @importFrom magrittr %>% 3 | "_PACKAGE" 4 | 5 | # The following block is used by usethis to automatically manage 6 | # roxygen namespace tags. Modify with care! 7 | ## usethis namespace: start 8 | ## usethis namespace: end 9 | if (getRversion() >= "2.15.1") { 10 | utils::globalVariables(".") 11 | } 12 | -------------------------------------------------------------------------------- /R/use.R: -------------------------------------------------------------------------------- 1 | #' Initiate {touchstone} 2 | #' 3 | #' This function will initialize {touchstone} in your package repository, use 4 | #' from root directory. 5 | #' 6 | #' @inheritParams use_touchstone_workflows 7 | #' @details 8 | #' For more information see the 'Using touchstone' vignette: 9 | #' `vignette("touchstone", package = "touchstone") 10 | #' @return 11 | #' The function is called for its side effects and returns `NULL` (invisibly). 12 | #' @examples 13 | #' \dontrun{ 14 | #' # within your repository 15 | #' use_touchstone() 16 | #' } 17 | #' @seealso [touchstone::use_touchstone_workflows()] 18 | #' @export 19 | use_touchstone <- function(overwrite = FALSE, 20 | command = NULL, 21 | limit_to = c("OWNER", "MEMBER", "COLLABORATOR"), 22 | force_upstream = TRUE) { 23 | dir_touchstone <- dir_touchstone() 24 | fs::dir_create(dir_touchstone) 25 | has_written_script <- copy_if_not_exists( 26 | system.file("script.R", package = "touchstone"), 27 | path_touchstone_script(), 28 | overwrite 29 | ) 30 | 31 | copy_if_not_exists( 32 | system.file("header.R", package = "touchstone"), 33 | fs::path(dir_touchstone, "header.R"), 34 | overwrite 35 | ) 36 | 37 | copy_if_not_exists( 38 | system.file("footer.R", package = "touchstone"), 39 | fs::path(dir_touchstone, "footer.R"), 40 | overwrite 41 | ) 42 | 43 | copy_if_not_exists( 44 | system.file("config.json", package = "touchstone"), 45 | fs::path(dir_touchstone, "config.json"), 46 | overwrite 47 | ) 48 | 49 | copy_if_not_exists( 50 | system.file("gitignore", package = "touchstone"), 51 | fs::path(dir_touchstone, ".gitignore"), 52 | overwrite 53 | ) 54 | 55 | use_touchstone_workflows( 56 | overwrite = overwrite, 57 | command = command, 58 | limit_to = limit_to, 59 | force_upstream = force_upstream 60 | ) 61 | 62 | append_rbuildignore("touchstone") 63 | 64 | if (has_written_script) { 65 | cli::cli_ul( 66 | "Replace the mtcars sample code in `touchstone/script.R` with code from your package you want to benchmark." 67 | ) 68 | } 69 | 70 | cli::cli_alert_info( 71 | "You can modify the PR comment, see {.code ?touchstone::pr_comment}." 72 | ) 73 | 74 | cli::cli_ul(paste0( 75 | "Commit and push to GitHub to the default branch to activate the ", 76 | "workflow, then ", 77 | ifelse(!is.null(command), "comment '{command}' on", "make"), 78 | " a pull request to trigger your first benchmark run." 79 | )) 80 | invisible(NULL) 81 | } 82 | 83 | 84 | copy_if_not_exists <- function(path, new_path, overwrite = FALSE) { 85 | if (!fs::file_exists(new_path) || overwrite) { 86 | fs::file_copy( 87 | path, 88 | new_path, 89 | overwrite 90 | ) 91 | cli::cli_alert_success("Populated file {.file {fs::path_file(new_path)}} in {.path {fs::path_dir(new_path)}/}.") 92 | TRUE 93 | } else { 94 | cli::cli_warn(paste0( 95 | "File {.file {fs::path_file(new_path)}} already exists", 96 | " at {.path {fs::path_dir(new_path)}/}, not copying." 97 | )) 98 | FALSE 99 | } 100 | } 101 | 102 | #' Use touchstone GitHub Actions Workflows 103 | #' 104 | #' This function will add (or update) the {touchstone} GitHub Actions workflows 105 | #' to your package repository. Use in the root directory of your repository. 106 | #' This function will be called by [touchstone::use_touchstone()], you should 107 | #' only need to call it to update the workflows or change their parameters. 108 | #' @param overwrite Overwrites files if they exist. 109 | #' @param command If set to `NULL` (the default) will run the workflow on every 110 | #' commit. If set to a command (e.g. `/benchmark`) the benchmark will only run 111 | #' when triggered with a comment on the PR starting with the command. 112 | #' @param limit_to Roles that are allowed to trigger the benchmark workflow 113 | #' via comment. See details for a list of roles and their definition. 114 | #' Set to `NULL` to allow everyone to trigger a benchmark. 115 | #' @param force_upstream Always benchmark against the upstream base branch. 116 | #' @return 117 | #' The function is called for its side effects and returns `NULL` (invisibly). 118 | #' @details 119 | #' Possible roles for `limit_to`: 120 | #' - `OWNER`: Owner of the repository, e.g. user for user/repo. 121 | #' - It is undocumented who holds this status in an org. 122 | #' - `MEMBER`: Member of org for org/repo. 123 | #' - `COLLABORATOR`: Anyone who was added as a collaborator to a repository. 124 | #' - `CONTRIBUTOR`: Anyone who has contributed any commit to the repository. 125 | #' 126 | #' Each user has only one role and the check does not interpolate permissions, 127 | #' so you have to add all roles whom you want to have permission to start the 128 | #' benchmark. So if you only add "COLLABORATOR" the owner will not be able to 129 | #' start the benchmark. 130 | #' 131 | #' GitHub will recognize additional, mostly unusual roles, see the 132 | #' [documentation](https://docs.github.com/en/rest/issues/comments). 133 | #' @export 134 | use_touchstone_workflows <- function(overwrite = FALSE, 135 | command = NULL, 136 | limit_to = c("OWNER", "MEMBER", "COLLABORATOR"), 137 | force_upstream = TRUE) { 138 | template <- readLines( 139 | system.file("touchstone-receive.yaml", package = "touchstone") 140 | ) 141 | 142 | force <- ifelse(force_upstream, "\n force_upstream: true", "") 143 | 144 | if (is.null(command)) { 145 | author_association <- "github.event.pull_request.author_association" 146 | } else { 147 | author_association <- "github.event.comment.author_association" 148 | } 149 | 150 | if (!is.null(limit_to)) { 151 | limit <- glue::glue_collapse( 152 | glue::glue(" {author_association} == '{limit_to}' "), 153 | sep = "||\n" 154 | ) 155 | limit <- glue::glue( 156 | "&&\n", 157 | " (\n", 158 | "{limit}\n", 159 | " )" 160 | ) 161 | } else { 162 | limit <- "" 163 | } 164 | 165 | 166 | if (is.null(command)) { 167 | trigger <- "\n pull_request:" 168 | ward <- glue::glue( 169 | "\n if:\n", 170 | " true ", 171 | limit, 172 | .trim = FALSE 173 | ) 174 | } else { 175 | # these have to be indented with 2 spaces per tab, 176 | # yaml does not allow tabs 177 | trigger <- glue::glue( 178 | "\n issue_comment:\n", 179 | " types: ['created', 'edited']", 180 | .trim = FALSE 181 | ) 182 | 183 | ward <- glue::glue( 184 | "\n if:\n", 185 | " github.event.issue.pull_request &&\n", 186 | " startsWith(github.event.comment.body, '{command}') ", 187 | limit, 188 | .trim = FALSE 189 | ) 190 | } 191 | 192 | # without as.character an additional newline is added 193 | wf <- sub("#- trigger", as.character(trigger), template) 194 | wf <- sub("#- ward", as.character(ward), wf) 195 | wf <- sub("#- force", as.character(force), wf) 196 | 197 | temp_wf <- fs::file_temp("receive.yml") 198 | writeLines(wf, temp_wf) 199 | 200 | workflows <- fs::dir_create(fs::path(".github", "workflows")) 201 | copy_if_not_exists( 202 | temp_wf, 203 | fs::path(workflows, "touchstone-receive.yaml"), 204 | overwrite 205 | ) 206 | 207 | copy_if_not_exists( 208 | system.file("touchstone-comment.yaml", package = "touchstone"), 209 | fs::path(workflows, "touchstone-comment.yaml"), 210 | overwrite 211 | ) 212 | 213 | invisible(NULL) 214 | } 215 | -------------------------------------------------------------------------------- /R/utils.R: -------------------------------------------------------------------------------- 1 | #' Touchstone managers 2 | #' 3 | #' Utilities to manage the touchstone database. 4 | #' @name touchstone_managers 5 | NULL 6 | 7 | #' @describeIn touchstone_managers returns the directory where the touchstone 8 | #' database lives. 9 | #' @aliases touchstone_managers 10 | #' @return 11 | #' Character vector of length one with th path to the touchstone directory. 12 | #' @export 13 | dir_touchstone <- function() { 14 | getOption("touchstone.dir", "touchstone") 15 | } 16 | 17 | #' Get the branch from the environment variable or fail if not set 18 | #' 19 | #' This function is only exported because it is a default argument. 20 | #' @param var The environment variable to retrieve. 21 | #' @return 22 | #' Returns a character vector of length one with the `branch` retrieved from the 23 | #' environment variable `var`. 24 | #' @export 25 | branch_get_or_fail <- function(var) { 26 | retrieved <- Sys.getenv(var) 27 | if (!nzchar(retrieved)) { 28 | if (rlang::is_interactive()) { 29 | cli::cli_alert_info(c(paste0( 30 | "touchstone not activated. Most likely, you want to run ", 31 | "{.code touchstone::activate()} and the below error should go away." 32 | ))) 33 | } 34 | cli::cli_abort(c( 35 | paste0( 36 | "If you don't specify the argument {.arg branch(s)}, you must set the environment ", 37 | "variable {.envvar {var}} to tell {.pkg touchstone} ", 38 | "which branches you want to benchmark against each other." 39 | ), 40 | "i" = "See {.code ?touchstone::run_script}." 41 | )) 42 | } else { 43 | retrieved 44 | } 45 | } 46 | 47 | path_touchstone_script <- function() { 48 | fs::path(dir_touchstone(), "script.R") 49 | } 50 | 51 | #' @describeIn touchstone_managers clears the touchstone database. 52 | #' @aliases touchstone_managers 53 | #' @param all Whether to clear the whole touchstone directory or just the 54 | #' records sub directory. 55 | #' @return 56 | #' The deleted paths (invisibly). 57 | #' @export 58 | touchstone_clear <- function(all = FALSE) { 59 | paths <- fs::path(dir_touchstone(), if (!all) c("records", "lib") else "") 60 | 61 | paths <- paths[fs::dir_exists(paths)] 62 | fs::dir_delete(paths) 63 | } 64 | 65 | #' Samples `branch` 66 | #' 67 | #' A block is a permutation of all unique elements in `branch`. Then, we sample 68 | #' `n` blocks. This is better than repeating one sample a certain number of 69 | #' times because if compute resources steadily increase, the first sample will 70 | #' always perform worse than the second, so the order within the blocks must be 71 | #' random. 72 | #' @keywords internal 73 | branches_upsample <- function(branch, n = 20) { 74 | purrr::map_dfr( 75 | rlang::seq2(1, n), 76 | ~ tibble::tibble(block = .x, branch = sample(unique(branch))) 77 | ) 78 | } 79 | 80 | ensure_dir <- function(...) { 81 | fs::dir_create(...) 82 | } 83 | 84 | schema_disk <- function() { 85 | c( 86 | elapsed = "numeric", iteration = "integer", branch = "character", 87 | block = "integer", 88 | name = "character" 89 | ) 90 | } 91 | 92 | 93 | local_git_checkout <- function(branch, 94 | path_pkg = ".", 95 | envir = parent.frame()) { 96 | current_branch <- gert::git_branch(repo = path_pkg) 97 | withr::defer( 98 | { 99 | gert::git_branch_checkout(current_branch, repo = path_pkg) 100 | cli::cli_alert_success("Switched back to branch {.val {current_branch}}.") 101 | }, 102 | envir = envir 103 | ) 104 | if (!(branch %in% gert::git_branch_list(repo = path_pkg)$name)) { 105 | cli::cli_abort("Branch {.val {branch}} does not exist, create it and add commits before you can switch on it.") 106 | } 107 | gert::git_branch_checkout(branch, repo = path_pkg) 108 | cli::cli_alert_success("Temporarily checked out branch {.val {branch}}.") 109 | } 110 | 111 | 112 | #' Temporarily remove all touchstone libraries from the path 113 | #' 114 | #' This is useful in conjunction with [run_script()]. 115 | #' @param path_pkg The path to the package that contains the touchstone library. 116 | #' @param envir The environment that triggers the deferred action on 117 | #' destruction. 118 | #' @details 119 | #' * Add a touchstone library to the path with [run_script()] and 120 | #' run a script. The script hence may contain calls to libraries only 121 | #' installed in touchstone libraries. 122 | #' * benchmark code with [benchmark_run()]. At the start, remove all 123 | #' all touchstone libraries from path and add the touchstone library we need. 124 | #' 125 | #' Advantages: Keep benchmarked repo in touchstone library only. 126 | #' @keywords internal 127 | local_without_touchstone_lib <- function(path_pkg = ".", envir = parent.frame()) { 128 | all <- normalizePath(.libPaths()) 129 | is_touchstone <- fs::path_has_parent( 130 | all, normalizePath(fs::path_abs(dir_touchstone()), mustWork = FALSE) 131 | ) 132 | all_but_touchstone <- all[!is_touchstone] 133 | withr::local_libpaths(all_but_touchstone, .local_envir = envir) 134 | } 135 | 136 | 137 | #' Make sure there is no installation of the package to benchmark in the global 138 | #' package library 139 | #' @keywords internal 140 | assert_no_global_installation <- function(path_pkg = ".") { 141 | local_without_touchstone_lib() 142 | check <- is_installed(path_pkg) 143 | if (check$installed) { 144 | cli::cli_abort(c( 145 | "Package {.pkg {check$name}} can be found on a non-touchstone library path. ", 146 | "!" = paste0( 147 | "This should not be the case - as the package should be installed in ", 148 | "dedicated library paths for benchmarking." 149 | ), 150 | "*" = 'To uninstall use {.code remove.packages("{check$name}", lib = "{ .libPaths()[1]}")}.' 151 | )) 152 | } 153 | } 154 | 155 | 156 | #' Check if a package is installed and unloading it 157 | #' @keywords internal 158 | is_installed <- function(path_pkg = ".") { 159 | path_desc <- fs::path(path_pkg, "DESCRIPTION") 160 | pkg_name <- unname(read.dcf(path_desc)[, "Package"]) 161 | list( 162 | name = pkg_name, 163 | installed = pkg_name %in% rownames(utils::installed.packages()) 164 | ) 165 | } 166 | 167 | is_windows <- function() { 168 | identical(.Platform$OS.type, "windows") 169 | } 170 | 171 | #' Pin asset directory 172 | #' 173 | #' Pin files or directories that need to be available on both branches when 174 | #' running the [touchstone_script]. During [benchmark_run()] they will 175 | #' available via [path_pinned_asset()]. This is only possible for assets 176 | #' *within* the git repository. 177 | #' @param ... Any number of directories or files, as strings, that you want to 178 | #' access in your [touchstone_script]. 179 | #' @param branch The branch the passed assets are copied from. 180 | #' @inheritParams fs::path 181 | #' @inheritParams fs::dir_copy 182 | #' @details When passing nested directories or files within nested directories 183 | #' the path will be copied recursively. See examples. 184 | #' @return The asset directory invisibly. 185 | #' @examples 186 | #' \dontrun{ 187 | #' # In the touchstone script within the repo "/home/user/pkg" 188 | #' 189 | #' pin_assets(c("/home/user/pkg/bench", "inst/setup.R", "some/nested/dir")) 190 | #' 191 | #' source(path_pinned_asset("inst/setup.R")) 192 | #' load(path_pinned_asset("some/nested/dir/data.RData")) 193 | #' 194 | #' touchstone::benchmark_run( 195 | #' expr_before_benchmark = { 196 | #' !!setup 197 | #' source(path_pinned_asset("bench/exprs.R")) 198 | #' }, 199 | #' run_me = some_exprs(), 200 | #' n = 6 201 | #' ) 202 | #' } 203 | #' @export 204 | pin_assets <- function(..., 205 | branch = branch_get_or_fail("GITHUB_HEAD_REF"), 206 | overwrite = TRUE) { 207 | asset_dir <- get_asset_dir(branch) 208 | 209 | local_git_checkout(branch) 210 | dirs <- rlang::list2(...) %>% unlist() 211 | valid_dirs <- dirs %>% purrr::map_lgl(fs::file_exists) 212 | 213 | if (!all(valid_dirs)) { 214 | cli::cli_warn(paste0( 215 | "The following asset{?s} could not be found and will ", 216 | "not be copied: {.path {dirs[!valid_dirs]}}" 217 | )) 218 | } 219 | 220 | create_and_copy <- function(asset) { 221 | git_root <- fs::path_real(get_git_root()) 222 | 223 | asset <- fs::path_real(asset) 224 | if (!fs::path_has_parent(asset, git_root)) { 225 | cli::cli_abort(c( 226 | "Can only pin assets within the git repository!", 227 | "i" = "Current repo: {.path {git_root}}" 228 | )) 229 | } 230 | 231 | rel_asset <- fs::path_rel(asset, git_root) 232 | new_path <- fs::path_join(c(asset_dir, rel_asset)) 233 | 234 | if (fs::is_dir(asset)[[1]]) { 235 | fs::dir_copy( 236 | asset, 237 | new_path, 238 | overwrite = overwrite 239 | ) 240 | } else if (fs::is_file(asset)[[1]]) { 241 | fs::path_dir(rel_asset) %>% 242 | fs::path(asset_dir, .) %>% 243 | fs::dir_create() 244 | 245 | fs::file_copy( 246 | asset, 247 | new_path, 248 | overwrite = overwrite 249 | ) 250 | } 251 | } 252 | 253 | dirs[valid_dirs] %>% purrr::walk(create_and_copy) 254 | 255 | if (any(valid_dirs)) { 256 | cli::cli_alert_success( 257 | paste0( 258 | "Pinned the following asset{?s} ", 259 | "to make {?it/them} available across branch checkouts: ", 260 | "{.path {dirs[valid_dirs]}}" 261 | ) 262 | ) 263 | } else { 264 | cli::cli_abort("No valid assets found.") 265 | } 266 | 267 | invisible(asset_dir) 268 | } 269 | 270 | #' Get path to asset 271 | #' 272 | #' Get the path to a pinned asset within a [touchstone_script]. 273 | #' @inheritParams fs::path 274 | #' @param branch The branch the passed asset was copied from. 275 | #' @return The absolute path to the asset. 276 | #' @seealso [pin_assets()] 277 | #' @export 278 | path_pinned_asset <- function(..., 279 | branch = branch_get_or_fail("GITHUB_HEAD_REF")) { 280 | asset_dir <- get_asset_dir(branch) 281 | 282 | path <- fs::path(asset_dir, ...) 283 | if (!fs::file_exists(path)) { 284 | cli::cli_abort("Asset {.path {fs::path(...)}} not pinned at {.val {branch}}.") 285 | } 286 | 287 | path 288 | } 289 | 290 | local_asset_dir <- function(..., env = parent.frame()) { 291 | branches <- rlang::list2(...) 292 | opts <- purrr::map(branches, fs::path_temp) 293 | names(opts) <- purrr::map_chr(branches, ~ paste0("touchstone.dir_assets_", .x)) 294 | withr::local_options(opts, .local_envir = env) 295 | 296 | invisible(opts) 297 | } 298 | 299 | get_asset_dir <- function(branch) { 300 | asset_dir <- getOption(paste0("touchstone.dir_assets_", branch)) 301 | 302 | if (is.null(asset_dir)) { 303 | cli::cli_abort(c( 304 | "Temporary directory for branch {.arg {branch}} not found. ", 305 | "i" = paste0( 306 | "This function is only for use within the {.code ?touchstone_script},", 307 | " which must be called with {.fun run_script}", 308 | "or after running {.fun touchstone::activate}" 309 | ) 310 | )) 311 | } 312 | 313 | asset_dir 314 | } 315 | 316 | 317 | #' @describeIn touchstone_managers returns the path to the file containing the pr comment. 318 | #' @aliases touchstone_managers 319 | #' @return 320 | #' Character vector of length one with the path to the pr comment. 321 | #' @export 322 | #' @seealso [pr_comment] 323 | path_pr_comment <- function() { 324 | fs::path(dir_touchstone(), "pr-comment/info.txt") 325 | } 326 | 327 | 328 | abort_string <- function() { 329 | rlang::abort(paste0( 330 | "Using a string as the named expression to benchmark or ", 331 | "`expr_before_benchmark` is deprecated." 332 | )) 333 | } 334 | 335 | append_rbuildignore <- function(dir) { 336 | ignore <- ".Rbuildignore" 337 | dir_str <- glue::glue("^{dir}$") 338 | 339 | if (fs::file_exists(ignore)) { 340 | already_ignored <- any(readLines(ignore) == dir_str) 341 | 342 | if (!already_ignored) { 343 | cat( 344 | dir_str, 345 | sep = "\n", file = ignore, append = TRUE 346 | ) 347 | } 348 | cli::cli_alert_success("Added {.path {dir}} to {.file {ignore}}.") 349 | } else { 350 | cli::cli_alert_warning( 351 | "Could not find {.file {ignore}} to add {.path {dir}}." 352 | ) 353 | } 354 | } 355 | 356 | find_git_root <- function(path = ".") { 357 | tryCatch( 358 | gert::git_find(path), 359 | error = function(err) { 360 | cli::cli_alert_danger( 361 | "Could not find git repository from current working directory!" 362 | ) 363 | cli::cli_alert_info( 364 | "Please manually set the option {.val touchstone.git_root}." 365 | ) 366 | NULL 367 | } 368 | ) 369 | } 370 | 371 | get_git_root <- function() { 372 | git_root <- getOption("touchstone.git_root") 373 | 374 | if (is.null(git_root)) { 375 | cli::cli_abort(c("Option {.val touchstone.git_root} not set!", 376 | "i" = 'Set it with {.code options(touchstone.git_root = "path to repo")}' 377 | )) 378 | } 379 | 380 | git_root 381 | } 382 | 383 | gh_cat <- function(string) { 384 | if (envvar_true("GITHUB_ACTIONS")) { 385 | cat(string) 386 | } 387 | } 388 | 389 | #' @importFrom rlang "%|%" 390 | envvar_true <- function(var) { 391 | stopifnot(is.character(var)) 392 | as.logical(toupper(Sys.getenv(var))) %|% FALSE 393 | } 394 | -------------------------------------------------------------------------------- /R/zzz.R: -------------------------------------------------------------------------------- 1 | .onLoad <- function(libname, pkgname) { 2 | op <- options() 3 | cache <- tibble::tibble( 4 | branch = character(), md5_hashes = list(), path_pkg = character() 5 | ) 6 | 7 | op_touchstone <- list( 8 | "touchstone.skip_install" = FALSE, 9 | "touchstone.git_root" = find_git_root(), 10 | "touchstone.dir" = "touchstone", 11 | # how many times should inner loop be ran in benchmark_run_iteration 12 | "touchstone.n_iterations" = 1, 13 | "touchstone.hash_source_package" = cache 14 | ) 15 | 16 | toset <- !(names(op_touchstone) %in% names(op)) 17 | if (any(toset)) options(op_touchstone[toset]) 18 | invisible() 19 | } 20 | -------------------------------------------------------------------------------- /README.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | output: github_document 3 | --- 4 | 5 | [![Lifecycle: 6 | experimental](https://img.shields.io/badge/lifecycle-experimental-orange.svg)](https://www.tidyverse.org/lifecycle/#experimental) 7 | [![R build 8 | status](https://github.com/lorenzwalthert/touchstone/workflows/R-CMD-check/badge.svg)](https://github.com/lorenzwalthert/touchstone/actions) 9 | 10 | 11 | ```{r, include = FALSE} 12 | knitr::opts_chunk$set( 13 | collapse = TRUE, 14 | comment = "#>", 15 | fig.path = "man/figures/README-", 16 | out.width = "100%", 17 | eval = FALSE 18 | ) 19 | ``` 20 | 21 | # touchstone 22 | 23 | {touchstone} is a developer tool for continuous benchmarking with a focus 24 | on reliable relative measurement, uncertainty reporting and user convenience. 25 | The results are directly reported as a comment in GitHub Pull Requests. 26 | 27 | ![](man/figures/screenshot-pr-comment.png) 28 | 29 | ## Installation 30 | 31 | You can install the package from CRAN: 32 | 33 | ``` r 34 | install.packages("touchstone") 35 | ``` 36 | 37 | And the development version from [GitHub](https://github.com/lorenzwalthert/touchstone){target="_blank"} with: 38 | 39 | ``` r 40 | # install.packages("devtools") 41 | devtools::install_github("lorenzwalthert/touchstone") 42 | ``` 43 | 44 | ## Getting Started 45 | You can start using {touchstone} in your package repository with: 46 | ```{r, eval = FALSE} 47 | touchstone::use_touchstone() 48 | ``` 49 | For a detailed explanation on how to configure and use {touchstone} see the 50 | ["Using touchstone"](https://lorenzwalthert.github.io/touchstone/articles/touchstone.html) vignette. 51 | 52 | ## Motivation 53 | 54 | The motivation for touchstone is to provide accurate benchmarking results for 55 | package developers. The following insights were the motivation: 56 | 57 | - **Compute power in GitHub Action VMs varies too much for reliable isolated 58 | benchmarks**: Experience with styler showed that a variation [around 59 | 30%](https://github.com/r-lib/styler/pull/679) for identical benchmarking code 60 | is possible. The solution to this is to benchmark the two branches in one 61 | CI/CD run and look at *relative difference* between branches. This matters in 62 | particular when running one iteration of a benchmark takes long (>> a few 63 | seconds) and speed implications are not huge. 64 | 65 | - **Timelines of absolute changes are mostly noise:** Maintaining a timeline of 66 | absolute benchmark times is of limited use because of the first bullet, at 67 | least when benchmark results don't vary significantly (> 50%). 68 | 69 | - **Dependencies should be identical across versions:** R and package versions 70 | of dependencies must be fixed via [RSPM](http://packagemanager.rstudio.com) to 71 | allow as much continuation as possible anyways. Changing the timestamp of RSPM 72 | can happen in PRs that are only dedicated to dependency updates. 73 | 74 | - **Pull requests are a natural unit for measuring speed:** Linking benchmarking 75 | to pull requests make sense because you can easily benchmark any revision 76 | against any other. Just open a pull request from one branch on another. You 77 | can also easily keep track of how performance of your development version 78 | evolves by opening a PR against a branch with the released version. No need to 79 | ever merge these. Once you release a new version of you package, you can 80 | change the target branch of the pull request to start anew. The pull request 81 | comments will preserve the information. 82 | 83 | ## Conceptual 84 | 85 | For your PR branch and the target branch, {touchstone} will: 86 | 87 | * build the two versions of the package in isolated libraries. 88 | 89 | * run the code you want to benchmark, many times, in [random 90 | order](https://lorenzwalthert.github.io/touchstone/articles/inference.html#sampling). 91 | This ensures the accurate measurement of relative differences between the 92 | branches. 93 | 94 | Once done with the measurements, it will 95 | 96 | * comment the results of the benchmarking on the PR. 97 | 98 | * create visualizations as Github Action artifacts that show how the 99 | distribution of the timings for both branches. 100 | 101 | ## Status 102 | 103 | This package is experimental. You can see an example usage in 104 | [{styler}](https://github.com/r-lib/styler/pull/799). 105 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | [![Lifecycle: 5 | experimental](https://img.shields.io/badge/lifecycle-experimental-orange.svg)](https://www.tidyverse.org/lifecycle/#experimental) 6 | [![R build 7 | status](https://github.com/lorenzwalthert/touchstone/workflows/R-CMD-check/badge.svg)](https://github.com/lorenzwalthert/touchstone/actions) 8 | 9 | 10 | # touchstone 11 | 12 | {touchstone} is a developer tool for continuous benchmarking with a 13 | focus on reliable relative measurement, uncertainty reporting and user 14 | convenience. The results are directly reported as a comment in GitHub 15 | Pull Requests. 16 | 17 | ![](man/figures/screenshot-pr-comment.png) 18 | 19 | ## Installation 20 | 21 | You can install the package from CRAN: 22 | 23 | ``` r 24 | install.packages("touchstone") 25 | ``` 26 | 27 | And the development version from 28 | GitHub 29 | with: 30 | 31 | ``` r 32 | # install.packages("devtools") 33 | devtools::install_github("lorenzwalthert/touchstone") 34 | ``` 35 | 36 | ## Getting Started 37 | 38 | You can start using {touchstone} in your package repository with: 39 | 40 | ``` r 41 | touchstone::use_touchstone() 42 | ``` 43 | 44 | For a detailed explanation on how to configure and use {touchstone} see 45 | the [“Using 46 | touchstone”](https://lorenzwalthert.github.io/touchstone/articles/touchstone.html) 47 | vignette. 48 | 49 | ## Motivation 50 | 51 | The motivation for touchstone is to provide accurate benchmarking 52 | results for package developers. The following insights were the 53 | motivation: 54 | 55 | - **Compute power in GitHub Action VMs varies too much for reliable 56 | isolated benchmarks**: Experience with styler showed that a 57 | variation [around 30%](https://github.com/r-lib/styler/pull/679) for 58 | identical benchmarking code is possible. The solution to this is to 59 | benchmark the two branches in one CI/CD run and look at *relative 60 | difference* between branches. This matters in particular when 61 | running one iteration of a benchmark takes long (>> a few 62 | seconds) and speed implications are not huge. 63 | 64 | - **Timelines of absolute changes are mostly noise:** Maintaining a 65 | timeline of absolute benchmark times is of limited use because of 66 | the first bullet, at least when benchmark results don’t vary 67 | significantly (> 50%). 68 | 69 | - **Dependencies should be identical across versions:** R and package 70 | versions of dependencies must be fixed via 71 | [RSPM](http://packagemanager.rstudio.com) to allow as much 72 | continuation as possible anyways. Changing the timestamp of RSPM can 73 | happen in PRs that are only dedicated to dependency updates. 74 | 75 | - **Pull requests are a natural unit for measuring speed:** Linking 76 | benchmarking to pull requests make sense because you can easily 77 | benchmark any revision against any other. Just open a pull request 78 | from one branch on another. You can also easily keep track of how 79 | performance of your development version evolves by opening a PR 80 | against a branch with the released version. No need to ever merge 81 | these. Once you release a new version of you package, you can change 82 | the target branch of the pull request to start anew. The pull 83 | request comments will preserve the information. 84 | 85 | ## Conceptual 86 | 87 | For your PR branch and the target branch, {touchstone} will: 88 | 89 | - build the two versions of the package in isolated libraries. 90 | 91 | - run the code you want to benchmark, many times, in [random 92 | order](https://lorenzwalthert.github.io/touchstone/articles/inference.html#sampling). 93 | This ensures the accurate measurement of relative differences 94 | between the branches. 95 | 96 | Once done with the measurements, it will 97 | 98 | - comment the results of the benchmarking on the PR. 99 | 100 | - create visualizations as Github Action artifacts that show how the 101 | distribution of the timings for both branches. 102 | 103 | ## Status 104 | 105 | This package is experimental. You can see an example usage in 106 | [{styler}](https://github.com/r-lib/styler/pull/799). 107 | -------------------------------------------------------------------------------- /_pkgdown.yml: -------------------------------------------------------------------------------- 1 | authors: 2 | Lorenz Walthert: 3 | href: https://lorenzwalthert.netlify.com 4 | 5 | articles: 6 | - title: Get started 7 | navbar: ~ 8 | contents: 9 | - touchstone 10 | - inference 11 | 12 | 13 | development: 14 | mode: auto 15 | 16 | reference: 17 | - title: "Getting started" 18 | desc: > 19 | Setup, interactive workflow and touchstone script execution. 20 | - contents: 21 | - use_touchstone 22 | - use_touchstone_workflows 23 | - activate 24 | - deactivate 25 | - run_script 26 | - title: "Scripting functionality" 27 | desc: > 28 | Functions you usually want to call in your touchstone script. 29 | - contents: 30 | - branch_install 31 | - benchmark_run 32 | - benchmark_analyze 33 | - title: "Pinning" 34 | desc: > 35 | Working with assets across branches. 36 | - contents: 37 | - pin_assets 38 | - path_pinned_asset 39 | - title: "Non-function documentation" 40 | desc: > 41 | Not functions, but worth documenting anyways. 42 | - contents: 43 | - touchstone_script 44 | - pr_comment 45 | - title: "i/o" 46 | desc: > 47 | Low-level tools for accessing the touchstone datastore. 48 | - contents: 49 | - benchmark_ls 50 | - benchmark_read 51 | - benchmark_write 52 | - dir_touchstone 53 | - touchstone_clear 54 | - path_pr_comment 55 | - title: "Misc" 56 | desc: > 57 | Miscellaneous. 58 | - contents: 59 | - branch_get_or_fail 60 | -------------------------------------------------------------------------------- /actions/README.md: -------------------------------------------------------------------------------- 1 | # Github Actions for {touchstone} 2 | 3 | This folder contains the [Github Actions](https://github.com/features/actions) used when benchmarking a package with {touchstone}. For [security reasons](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/) the workflow is split into twp separate actions: 4 | 5 | * [lorenzwalthert/touchstone/actions/receive](https://github.com/lorenzwalthert/touchstone/tree/main/actions/receive) 6 | * Reads `config.json` to prepare and run the benchmark job. 7 | * Does not have read & write access. 8 | * Started via PR/push to PR (on main branch). 9 | * [lorenzwalthert/touchstone/actions/comment](https://github.com/lorenzwalthert/touchstone/tree/main/actions/comment) 10 | * Comments the results on the PR that originated the workflow run. 11 | * Has read & write access. 12 | * Started automatically when receive job finishes. 13 | * Will create an additional commit status to the PR check suite. 14 | 15 | The actions will always be compatible with the version of {touchstone} of the same git ref, so we recommend using the same tag for both {touchstone} and the actions, e.g. with {touchstone} v0.0.1: 16 | ```yaml 17 | - uses lorenzwalthert/touchstone/actions/receive@v0.0.1 18 | ``` 19 | There is also the tag `v1` which will occasionally be moved forward for small changes but never for breaking changes. ****We recommend this version for pre-CRAN releases.*** 20 | ```yaml 21 | - uses lorenzwalthert/touchstone/actions/receive@v1 22 | ``` 23 | -------------------------------------------------------------------------------- /actions/comment/action.yaml: -------------------------------------------------------------------------------- 1 | name: 'comment' 2 | description: 'Action to comment {touchstone} results on the appropriate PR. Needs read/write access.' 3 | inputs: 4 | GITHUB_TOKEN: 5 | description: 'The GITHUB_TOKEN secret.' 6 | required: true 7 | runs: 8 | using: "composite" 9 | steps: 10 | - name: 'Download artifact' 11 | id: 'download' 12 | uses: actions/download-artifact@v4 13 | with: 14 | name: pr 15 | github-token: ${{ inputs.GITHUB_TOKEN }} 16 | repository: ${{ github.repository }} 17 | run-id: ${{github.event.workflow_run.id }} 18 | - name: 'Comment on PR' 19 | id: 'comment' 20 | uses: actions/github-script@v7 21 | with: 22 | github-token: ${{ inputs.GITHUB_TOKEN }} 23 | script: | 24 | var fs = require('fs'); 25 | var issue_number = Number(fs.readFileSync('./NR')); 26 | var body = fs.readFileSync('./info.txt').toString(); 27 | await github.rest.issues.createComment({ 28 | owner: context.repo.owner, 29 | repo: context.repo.repo, 30 | issue_number: issue_number, 31 | body: body 32 | }); 33 | - uses: actions/github-script@v7 34 | if: always() 35 | with: 36 | script: | 37 | let url = '${{ github.event.workflow_run.html_url }}' 38 | let any_failed = ${{ steps.comment.outcome == 'failure' || steps.download.outcome == 'failure' }} 39 | let state = 'success' 40 | let description = 'Commenting succeeded!' 41 | 42 | if(${{ github.event.workflow_run.conclusion == 'failure'}} || any_failed) { 43 | state = 'failure' 44 | description = 'Commenting failed!' 45 | if(any_failed) { 46 | url = "https://github.com/${{github.repository}}/actions/runs/" + 47 | "${{ github.run_id }}" 48 | } 49 | } 50 | 51 | github.rest.repos.createCommitStatus({ 52 | owner: context.repo.owner, 53 | repo: context.repo.repo, 54 | sha: '${{ github.event.workflow_run.head_sha}}', 55 | state: state, 56 | target_url: url, 57 | description: description, 58 | context: 'touchstone comment' 59 | }) 60 | -------------------------------------------------------------------------------- /actions/receive/action.yaml: -------------------------------------------------------------------------------- 1 | name: 'receive' 2 | description: 'Action to run {touchstone} benchmarks and upload the results.' 3 | inputs: 4 | force_upstream: 5 | description: 'Always compare against the upstream base branch when benchmarking PRs in forked repositories.' 6 | required: false 7 | default: false 8 | benchmarking_repo: 9 | description: 'Additional repository required for benchmarking.' 10 | required: false 11 | benchmarking_ref: 12 | description: 'Ref of benchmarking repository.' 13 | required: false 14 | benchmarking_path: 15 | description: 'Path to check out benchmarking repository to.' 16 | required: false 17 | cache-version: 18 | description: 'Integer to use as cache version. Increment to use new cache.' 19 | required: true 20 | default: 1 21 | touchstone_package: 22 | description: 'Which package of {touchstone} should be used. Mainly for debugging.' 23 | required: true 24 | default: 'github::lorenzwalthert/touchstone' 25 | touchstone_ref: 26 | description: 'Which branch or tag of {touchstone} should be used. This will be appended to the touchstone_package option. Mainly for debugging.' 27 | required: true 28 | default: '@v1' 29 | extra-packages: 30 | description: 'Any extra packages that should be installed.' 31 | extra-repositories: 32 | description: 'Specify extra repositories to be passed to setup-r' 33 | default: '' 34 | 35 | runs: 36 | using: "composite" 37 | steps: 38 | - uses: actions/github-script@v7 39 | id: get-pull 40 | with: 41 | script: | 42 | var pr = await github.rest.pulls.get({ 43 | owner: context.repo.owner, 44 | repo: context.repo.repo, 45 | pull_number: context.issue.number 46 | }) 47 | pr = pr.data 48 | 49 | var is_fork = pr.head.repo.fork && 50 | (pr.base.repo.clone_url !== pr.head.repo.clone_url || 51 | ${{ inputs.force_upstream }}) 52 | 53 | if (is_fork) { 54 | var head_repo = await github.rest.repos.get({ 55 | owner: pr.head.repo.owner.login, 56 | repo: pr.head.repo.name 57 | }) 58 | head_repo = head_repo.data 59 | pr.base.repo.clone_url = head_repo.parent.clone_url 60 | } 61 | 62 | var pull = { 63 | number: pr.number, 64 | head_repo: pr.head.repo.full_name, 65 | head_ref: pr.head.ref, 66 | is_fork: is_fork, 67 | base_git: pr.base.repo.clone_url, 68 | base_ref: pr.base.ref 69 | } 70 | 71 | console.log(pull) 72 | return pull 73 | - name: Set envvars 74 | shell: bash 75 | run: | 76 | echo "GITHUB_HEAD_REF=${{ fromJSON(steps.get-pull.outputs.result).head_ref }}" >> $GITHUB_ENV 77 | echo "GITHUB_BASE_REF=${{ fromJSON(steps.get-pull.outputs.result).base_ref }}" >> $GITHUB_ENV 78 | - name: Checkout repo 79 | uses: actions/checkout@v4 80 | with: 81 | repository: ${{ fromJSON(steps.get-pull.outputs.result).head_repo }} 82 | ref: ${{ fromJSON(steps.get-pull.outputs.result).head_ref }} 83 | fetch-depth: 0 84 | - name: Set up git user 85 | run: | 86 | git config --local user.name "GitHub Actions" 87 | git config --local user.email "actions@github.com" 88 | shell: bash 89 | - name: Get base branch 90 | shell: bash 91 | run: | 92 | REMOTE=origin 93 | if ${{ fromJSON(steps.get-pull.outputs.result).is_fork }}; then 94 | git remote add touchstone_base ${{ fromJSON(steps.get-pull.outputs.result).base_git }} 95 | git fetch touchstone_base 96 | REMOTE=touchstone_base 97 | fi 98 | git branch -f ${{ env.GITHUB_BASE_REF }} $REMOTE/${{ env.GITHUB_BASE_REF }} 99 | - name: Setup R 100 | uses: r-lib/actions/setup-r@v2 101 | with: 102 | extra-repositories: ${{ inputs.extra-repositories }} 103 | 104 | - name: Install dependencies 105 | uses: r-lib/actions/setup-r-dependencies@v2 106 | with: 107 | cache-version: ${{ inputs.cache_version }} 108 | extra-packages: | 109 | any::ggplot2 110 | any::dplyr 111 | any::gert 112 | any::glue 113 | ${{ inputs.touchstone_package }}${{ inputs.touchstone_ref }} 114 | ${{ inputs.extra-packages }} 115 | - name: Remove global installation 116 | run: | 117 | pkg <- unlist(read.dcf('DESCRIPTION')[, 'Package']) 118 | if (pkg %in% rownames(installed.packages())) { 119 | remove.packages(pkg) 120 | cat('removed package ', pkg, '.', sep = "") 121 | } 122 | shell: Rscript {0} 123 | - name: Checkout benchmarking repo 124 | if: ${{ inputs.benchmarking_repo != ''}} 125 | uses: actions/checkout@v4 126 | with: 127 | repository: ${{ inputs.benchmarking_repo }} 128 | ref: ${{ inputs.benchmarking_ref }} 129 | path: ${{ inputs.benchmarking_path }} 130 | - name: Run benchmarks 131 | run: | 132 | Sys.setenv( 133 | GITHUB_BASE_REF = "${{ env.GITHUB_BASE_REF }}", 134 | GITHUB_HEAD_REF = "${{ env.GITHUB_HEAD_REF }}" 135 | ) 136 | touchstone::run_script("touchstone/script.R") 137 | shell: Rscript {0} 138 | - name: Uploading Results 139 | run: | 140 | echo ${{ fromJSON(steps.get-pull.outputs.result).number }} > ./touchstone/pr-comment/NR 141 | cat touchstone/pr-comment/info.txt 142 | shell: bash 143 | - uses: actions/upload-artifact@v4 144 | with: 145 | name: visual-benchmarks 146 | path: touchstone/plots/ 147 | - uses: actions/upload-artifact@v4 148 | with: 149 | name: results 150 | path: touchstone/records/ 151 | - uses: actions/upload-artifact@v4 152 | with: 153 | name: pr 154 | path: touchstone/pr-comment/ 155 | -------------------------------------------------------------------------------- /inst/WORDLIST: -------------------------------------------------------------------------------- 1 | api 2 | aut 3 | ba 4 | bcea 5 | benchmarked 6 | benchmarking 7 | BugReports 8 | callr 9 | ci 10 | cli 11 | CMD 12 | config 13 | Config 14 | const 15 | cr 16 | cran 17 | cre 18 | createComment 19 | createCommitStatus 20 | datastore 21 | dcf 22 | de 23 | Dependabot 24 | deps 25 | desc 26 | dev 27 | devtools 28 | dir 29 | docType 30 | downloadArtifact 31 | dplyr 32 | env 33 | envir 34 | envvars 35 | eval 36 | expr 37 | fi 38 | fromJson 39 | fromJSON 40 | fs 41 | gcc 42 | gert 43 | getenv 44 | getRversion 45 | getwd 46 | ggplot 47 | github 48 | gitignore 49 | hashFiles 50 | href 51 | https 52 | icloud 53 | io 54 | jacob 55 | Jens 56 | json 57 | knitr 58 | LazyData 59 | lenth 60 | libcurl 61 | libgit 62 | libpaths 63 | LIBS 64 | Lifecycle 65 | linter 66 | linters 67 | linux 68 | listWorkflowRunArtifacts 69 | lorenz 70 | lorenzwalthert 71 | magrittr 72 | matchArtifact 73 | md 74 | MERCHANTABILITY 75 | mkdir 76 | navbar 77 | netlify 78 | NONINFRINGEMENT 79 | openssl 80 | orcid 81 | ORCID 82 | os 83 | packagemanager 84 | pak 85 | pkgapi 86 | pkgdown 87 | pre 88 | prepending 89 | PRs 90 | purrr 91 | quasiquotation 92 | rc 93 | Rds 94 | readFileSync 95 | README 96 | repo 97 | RHUB 98 | rightarrow 99 | rlang 100 | rmarkdown 101 | roclet 102 | roclets 103 | roxygen 104 | Roxygen 105 | RoxygenNote 106 | Rscript 107 | rspm 108 | RSPM 109 | rstudio 110 | runif 111 | saveRDS 112 | seealso 113 | sep 114 | setenv 115 | sha 116 | SHA 117 | sideeffects 118 | sprintf 119 | startsWith 120 | styfle 121 | styler 122 | sublicense 123 | subprocess 124 | sudo 125 | Sys 126 | sysreq 127 | sysreqs 128 | tada 129 | testpkg 130 | testthat 131 | th 132 | tibble 133 | tidyselect 134 | toString 135 | touchstone 136 | ubuntu 137 | unlist 138 | usethis 139 | va 140 | vctrs 141 | VignetteBuilder 142 | VMs 143 | walthert 144 | Walthert 145 | withr 146 | writeFileSync 147 | writeLines 148 | wujciak 149 | Wujciak 150 | yourpkg 151 | zzz 152 | -------------------------------------------------------------------------------- /inst/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "os": "ubuntu-20.04", 3 | "r": "4.1.1", 4 | "rspm": "https://packagemanager.rstudio.com/all/__linux__/focal/2022-01-07+MTo3NDQwNTcyLDI6NDUyNjIxNTs0QzU3NUZBRQ", 5 | // "benchmarking_repo": "lorenzwalthert/here", 6 | // "benchmarking_ref": "ca9c8e69c727def88d8ba1c8b85b0e0bcea87b3f", 7 | "benchmarking_path": "touchstone/sources/here", 8 | } 9 | -------------------------------------------------------------------------------- /inst/footer.R: -------------------------------------------------------------------------------- 1 | # You can modify the PR comment footer here. You can use github markdown e.g. 2 | # emojis like :tada:. 3 | # This file will be parsed and evaluate within the context of 4 | # `benchmark_analyze` and should return the comment text as the last value. 5 | # See `?touchstone::pr_comment` 6 | link <- "https://lorenzwalthert.github.io/touchstone/articles/inference.html" 7 | glue::glue( 8 | "\nFurther explanation regarding interpretation and", 9 | " methodology can be found in the [documentation]({link})." 10 | ) 11 | -------------------------------------------------------------------------------- /inst/gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !script.R 3 | !config.json 4 | !.gitignore 5 | !header.R 6 | !footer.R 7 | -------------------------------------------------------------------------------- /inst/header.R: -------------------------------------------------------------------------------- 1 | # You can modify the PR comment header here. You can use github markdown e.g. 2 | # emojis like :tada:. 3 | # This file will be parsed and evaluate within the context of 4 | # `benchmark_analyze` and should return the comment text as the last value. 5 | # Available variables for glue substitution: 6 | # * ci: confidence interval 7 | # * branches: BASE and HEAD branches benchmarked against each other. 8 | # See `?touchstone::pr_comment` 9 | glue::glue( 10 | "This is how benchmark results would change (along with a", 11 | " {100 * ci}% confidence interval in relative change) if ", 12 | "{system2('git', c('rev-parse', 'HEAD'), stdout = TRUE)} is merged into {branches[1]}:\n" 13 | ) 14 | -------------------------------------------------------------------------------- /inst/script.R: -------------------------------------------------------------------------------- 1 | # see `help(run_script, package = 'touchstone')` on how to run this 2 | # interactively 3 | 4 | # TODO OPTIONAL Add directories you want to be available in this file or during the 5 | # benchmarks. 6 | # touchstone::pin_assets("some/dir") 7 | 8 | # installs branches to benchmark 9 | touchstone::branch_install() 10 | 11 | # benchmark a function call from your package (two calls per branch) 12 | touchstone::benchmark_run( 13 | # expr_before_benchmark = source("dir/data.R"), #<-- TODO OTPIONAL setup before benchmark 14 | random_test = yourpkg::f(), #<- TODO put the call you want to benchmark here 15 | n = 2 16 | ) 17 | 18 | # TODO OPTIONAL benchmark any R expression (six calls per branch) 19 | # touchstone::benchmark_run( 20 | # more = { 21 | # if (TRUE) { 22 | # y <- yourpkg::f2(x = 3) 23 | # } 24 | # }, #<- TODO put the call you want to benchmark here 25 | # n = 6 26 | # ) 27 | 28 | 29 | # create artifacts used downstream in the GitHub Action 30 | touchstone::benchmark_analyze() 31 | -------------------------------------------------------------------------------- /inst/touchstone-comment.yaml: -------------------------------------------------------------------------------- 1 | name: Continuous Benchmarks (Comment) 2 | 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.head_ref }} 5 | cancel-in-progress: true 6 | 7 | on: 8 | workflow_run: 9 | workflows: ["Continuous Benchmarks (Receive)"] 10 | types: 11 | - completed 12 | 13 | permissions: 14 | contents: read 15 | statuses: write 16 | pull-requests: write 17 | 18 | jobs: 19 | upload: 20 | runs-on: ubuntu-latest 21 | if: > 22 | ${{ github.event.workflow_run.event == 'pull_request' }} 23 | steps: 24 | - uses: lorenzwalthert/touchstone/actions/comment@v1 25 | with: 26 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 27 | -------------------------------------------------------------------------------- /inst/touchstone-receive.yaml: -------------------------------------------------------------------------------- 1 | name: Continuous Benchmarks (Receive) 2 | 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.head_ref }} 5 | cancel-in-progress: true 6 | 7 | on: #- trigger 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | prepare: 14 | runs-on: ubuntu-latest #- ward 15 | outputs: 16 | config: ${{ steps.read_touchstone_config.outputs.config }} 17 | steps: 18 | - name: Checkout repo 19 | uses: actions/checkout@v3 20 | with: 21 | fetch-depth: 0 22 | 23 | - id: read_touchstone_config 24 | run: | 25 | content=`cat ./touchstone/config.json` 26 | # the following lines are only required for multi line json 27 | content="${content//'%'/'%25'}" 28 | content="${content//$'\n'/'%0A'}" 29 | content="${content//$'\r'/'%0D'}" 30 | # end of optional handling for multi line json 31 | echo "::set-output name=config::$content" 32 | build: 33 | needs: prepare 34 | runs-on: ${{ matrix.config.os }} 35 | strategy: 36 | fail-fast: false 37 | matrix: 38 | config: 39 | - ${{ fromJson(needs.prepare.outputs.config) }} 40 | env: 41 | R_REMOTES_NO_ERRORS_FROM_WARNINGS: true 42 | RSPM: ${{ matrix.config.rspm }} 43 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 44 | steps: 45 | - uses: lorenzwalthert/touchstone/actions/receive@v1 46 | with: 47 | cache-version: 1 48 | benchmarking_repo: ${{ matrix.config.benchmarking_repo }} 49 | benchmarking_ref: ${{ matrix.config.benchmarking_ref }} 50 | benchmarking_path: ${{ matrix.config.benchmarking_path }} #- force 51 | -------------------------------------------------------------------------------- /inst/workflow-visualization.odt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lorenzwalthert/touchstone/f17ea09bc1835dfc8705135ba406fec85c25aa17/inst/workflow-visualization.odt -------------------------------------------------------------------------------- /man/activate.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/source.R 3 | \name{activate} 4 | \alias{activate} 5 | \alias{deactivate} 6 | \title{Activate touchstone environment} 7 | \usage{ 8 | activate( 9 | head_branch = gert::git_branch(), 10 | base_branch = getOption("touchstone.default_base_branch", "main"), 11 | n = 1, 12 | env = parent.frame() 13 | ) 14 | 15 | deactivate(env = parent.frame()) 16 | } 17 | \arguments{ 18 | \item{head_branch}{Git branch to be used as the \code{GITHUB_HEAD_REF} branch 19 | (i.e. the branch with new changes) when running benchmarks. Defaults to the 20 | current branch.} 21 | 22 | \item{base_branch}{Git branch for the \code{GITHUB_BASE_REF} (i.e. the branch you 23 | want to merge your changes into) when running benchmarks. Defaults to 'main' 24 | if the option \code{touchstone.default_base_branch}is not set.} 25 | 26 | \item{n}{Number of times benchmarks should be run for each \code{branch}. Will 27 | override \code{n} argument in all interactive calls to \code{\link[=benchmark_run]{benchmark_run()}}.} 28 | 29 | \item{env}{In which environment the temporary changes should be made. 30 | For use within functions.} 31 | } 32 | \description{ 33 | This sets environment variables, R options and library paths to work 34 | interactively on the \link{touchstone_script}. 35 | } 36 | \section{Functions}{ 37 | \itemize{ 38 | \item \code{deactivate()}: Restore the original environment state. 39 | 40 | }} 41 | \examples{ 42 | \dontrun{ 43 | activate() 44 | # You can now test parts of your touchstone script, e.g. touchstone/script.R 45 | deactivate() 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /man/assert_no_global_installation.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{assert_no_global_installation} 4 | \alias{assert_no_global_installation} 5 | \title{Make sure there is no installation of the package to benchmark in the global 6 | package library} 7 | \usage{ 8 | assert_no_global_installation(path_pkg = ".") 9 | } 10 | \description{ 11 | Make sure there is no installation of the package to benchmark in the global 12 | package library 13 | } 14 | \keyword{internal} 15 | -------------------------------------------------------------------------------- /man/benchmark_analyze.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/analyze.R 3 | \name{benchmark_analyze} 4 | \alias{benchmark_analyze} 5 | \title{Turn raw benchmark results into text and figures} 6 | \usage{ 7 | benchmark_analyze( 8 | branches = c(branch_get_or_fail("GITHUB_BASE_REF"), 9 | branch_get_or_fail("GITHUB_HEAD_REF")), 10 | names = NULL, 11 | ci = 0.95 12 | ) 13 | } 14 | \arguments{ 15 | \item{branches}{The names of the branches for which analysis should be 16 | created.} 17 | 18 | \item{names}{The names of the benchmarks to analyze. If \code{NULL}, all 19 | benchmarks with the \code{branches} are taken.} 20 | 21 | \item{ci}{The confidence level, defaults to 95\%.} 22 | } 23 | \value{ 24 | A character vector that summarizes the benchmarking results. 25 | } 26 | \description{ 27 | Turn raw benchmark results into text and figures 28 | } 29 | \details{ 30 | Creates two side effects: 31 | \itemize{ 32 | \item Density plots for each element in \code{branches} are written to 33 | \code{touchstone/plots}. 34 | \item A text explaining the speed diff is written to 35 | \code{touchstone/pr-comment/info.txt} for every registered benchmarking 36 | expression. See \code{vignette("inference", package = "touchstone")} for 37 | details. 38 | } 39 | 40 | Requires \link[dplyr:dplyr-package]{dplyr::dplyr}, \link[ggplot2:ggplot2-package]{ggplot2::ggplot2} and \link[glue:glue]{glue::glue}. 41 | } 42 | -------------------------------------------------------------------------------- /man/benchmark_ls.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/io.R 3 | \name{benchmark_ls} 4 | \alias{benchmark_ls} 5 | \title{List which benchmarks were recorded} 6 | \usage{ 7 | benchmark_ls(name = "") 8 | } 9 | \arguments{ 10 | \item{name}{The name of the benchmark.} 11 | } 12 | \value{ 13 | A tibble with name and branches of the existing benchmarks. 14 | } 15 | \description{ 16 | List which benchmarks were recorded 17 | } 18 | -------------------------------------------------------------------------------- /man/benchmark_read.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/io.R 3 | \name{benchmark_read} 4 | \alias{benchmark_read} 5 | \title{Read benchmarks} 6 | \usage{ 7 | benchmark_read(name, branch) 8 | } 9 | \arguments{ 10 | \item{name}{The name of the benchmark.} 11 | 12 | \item{branch}{A character vector of length one to indicate the git branch (i.e. 13 | commit, tag, branch etc) of the benchmarking.} 14 | } 15 | \value{ 16 | A tibble with the benchmarks. 17 | } 18 | \description{ 19 | Read benchmarks 20 | } 21 | -------------------------------------------------------------------------------- /man/benchmark_run.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/core.R 3 | \name{benchmark_run} 4 | \alias{benchmark_run} 5 | \title{Run a benchmark for git branches} 6 | \usage{ 7 | benchmark_run( 8 | expr_before_benchmark = { 9 | }, 10 | ..., 11 | branches = c(branch_get_or_fail("GITHUB_BASE_REF"), 12 | branch_get_or_fail("GITHUB_HEAD_REF")), 13 | n = 100, 14 | path_pkg = "." 15 | ) 16 | } 17 | \arguments{ 18 | \item{expr_before_benchmark}{Expression to run before 19 | the benchmark is ran, will be captured with \code{\link[rlang:defusing-advanced]{rlang::enexpr()}}. So you can 20 | use quasiquotation.} 21 | 22 | \item{...}{Named expression of length one with code to benchmark, 23 | will be captured with \code{\link[rlang:defusing-advanced]{rlang::enexprs()}}. So you can use quasiquotation.} 24 | 25 | \item{branches}{Character vector with branch names to benchmark. The package 26 | must be built for each benchmarked branch beforehand with \code{\link[=branch_install]{branch_install()}}. 27 | The base branch is the target branch of the pull request in a workflow run, 28 | the head branch is the source branch of the pull request in a workflow run.} 29 | 30 | \item{n}{Number of times benchmarks should be run for each \code{branch}. The more 31 | iterations you run, the more narrow your confidence interval will be and 32 | the smaller the differences you will detect. See also 33 | \code{vignette("inference")}. To simplify interactive experimentation with 34 | \code{benchmark_run()}, \code{n} will be overridden in interactive usage after the 35 | user calls \code{activate(..., n = 1)}.} 36 | 37 | \item{path_pkg}{The path to the package to benchmark. Will be used to 38 | temporarily checkout the branch during benchmarking.} 39 | } 40 | \value{ 41 | All timings in a tibble. 42 | } 43 | \description{ 44 | Run a benchmark for git branches 45 | } 46 | \details{ 47 | Runs the following loop \code{n} times: 48 | \itemize{ 49 | \item removes all touchstone libraries from the library path, adding the one 50 | corresponding to \code{branch}. 51 | \item runs setup code \code{exp_before_branch}. 52 | \item benchmarks \code{expr_to_benchmark} and writes them to disk. 53 | } 54 | } 55 | \section{Caution}{ 56 | 57 | This function will perform various git operations that affect the state of 58 | the directory it is ran in, in particular different branches will be checked 59 | out. Ensure a clean git working directory before invocation. 60 | } 61 | 62 | -------------------------------------------------------------------------------- /man/benchmark_run_impl.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/core.R 3 | \name{benchmark_run_impl} 4 | \alias{benchmark_run_impl} 5 | \title{Checkout a branch from a repo and run an iteration} 6 | \usage{ 7 | benchmark_run_impl(branch, block, expr_before_benchmark, dots, path_pkg) 8 | } 9 | \arguments{ 10 | \item{branch}{A character vector of length one to indicate the git branch (i.e. 11 | commit, tag, branch etc) of the benchmarking.} 12 | 13 | \item{block}{All branches that appear once in a block.} 14 | 15 | \item{expr_before_benchmark}{Expression to run before 16 | the benchmark is ran, will be captured with \code{\link[rlang:defusing-advanced]{rlang::enexpr()}}. So you can 17 | use quasiquotation.} 18 | 19 | \item{dots}{list of quoted expressions (length 1).} 20 | 21 | \item{path_pkg}{The path to the root of the package you want to benchmark.} 22 | } 23 | \description{ 24 | Checkout a branch from a repo and run an iteration 25 | } 26 | \keyword{internal} 27 | -------------------------------------------------------------------------------- /man/benchmark_run_iteration.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/core.R 3 | \name{benchmark_run_iteration} 4 | \alias{benchmark_run_iteration} 5 | \title{Run a benchmark iteration} 6 | \usage{ 7 | benchmark_run_iteration( 8 | expr_before_benchmark, 9 | dots, 10 | branch, 11 | block, 12 | n = getOption("touchstone.n_iterations", 1) 13 | ) 14 | } 15 | \arguments{ 16 | \item{expr_before_benchmark}{Expression to run before 17 | the benchmark is ran, will be captured with \code{\link[rlang:defusing-advanced]{rlang::enexpr()}}. So you can 18 | use quasiquotation.} 19 | 20 | \item{dots}{list of quoted expressions (length 1).} 21 | 22 | \item{branch}{A character vector of length one to indicate the git branch (i.e. 23 | commit, tag, branch etc) of the benchmarking.} 24 | 25 | \item{block}{All branches that appear once in a block.} 26 | 27 | \item{n}{Number of iterations to run a benchmark within an iteration.} 28 | } 29 | \description{ 30 | Run a benchmark iteration 31 | } 32 | \keyword{internal} 33 | -------------------------------------------------------------------------------- /man/benchmark_verbalize.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/analyze.R 3 | \name{benchmark_verbalize} 4 | \alias{benchmark_verbalize} 5 | \title{Create nice text from benchmarks} 6 | \usage{ 7 | benchmark_verbalize(benchmark, timings, branches, ci) 8 | } 9 | \description{ 10 | \code{branches} must be passed because the order is relevant. 11 | } 12 | \keyword{internal} 13 | -------------------------------------------------------------------------------- /man/benchmark_write.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/io.R 3 | \name{benchmark_write} 4 | \alias{benchmark_write} 5 | \title{Write a benchmark} 6 | \usage{ 7 | benchmark_write( 8 | benchmark, 9 | name, 10 | branch, 11 | block = NA, 12 | iteration = NA, 13 | append = TRUE 14 | ) 15 | } 16 | \arguments{ 17 | \item{benchmark}{The result of \code{\link[bench:mark]{bench::mark()}}, with \code{iterations = 1}.} 18 | 19 | \item{name}{The name of the benchmark.} 20 | 21 | \item{branch}{A character vector of length one to indicate the git branch (i.e. 22 | commit, tag, branch etc) of the benchmarking.} 23 | 24 | \item{block}{All branches that appear once in a block.} 25 | 26 | \item{iteration}{An integer indicating to which iteration the benchmark 27 | refers to. Multiple iterations within a block always benchmark the same 28 | \code{branch}.} 29 | 30 | \item{append}{Whether to append the result to the file or not.} 31 | } 32 | \value{ 33 | Character vector of length one with path to the record written (invisibly). 34 | } 35 | \description{ 36 | Write a benchmark 37 | } 38 | -------------------------------------------------------------------------------- /man/branch_get_or_fail.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{branch_get_or_fail} 4 | \alias{branch_get_or_fail} 5 | \title{Get the branch from the environment variable or fail if not set} 6 | \usage{ 7 | branch_get_or_fail(var) 8 | } 9 | \arguments{ 10 | \item{var}{The environment variable to retrieve.} 11 | } 12 | \value{ 13 | Returns a character vector of length one with the \code{branch} retrieved from the 14 | environment variable \code{var}. 15 | } 16 | \description{ 17 | This function is only exported because it is a default argument. 18 | } 19 | -------------------------------------------------------------------------------- /man/branch_install.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/prepare.R 3 | \name{branch_install} 4 | \alias{branch_install} 5 | \title{Install branches} 6 | \usage{ 7 | branch_install( 8 | branches = c(branch_get_or_fail("GITHUB_BASE_REF"), 9 | branch_get_or_fail("GITHUB_HEAD_REF")), 10 | path_pkg = ".", 11 | install_dependencies = FALSE 12 | ) 13 | } 14 | \arguments{ 15 | \item{branches}{The names of the branches in a character vector.} 16 | 17 | \item{path_pkg}{The path to the repository to install.} 18 | 19 | \item{install_dependencies}{Passed to \code{\link[remotes:install_local]{remotes::install_local()}}.} 20 | } 21 | \value{ 22 | The global and touchstone library paths in a character vector (invisibly). 23 | } 24 | \description{ 25 | Installs each \code{branch} in a separate library for isolation. 26 | } 27 | -------------------------------------------------------------------------------- /man/branch_install_impl.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/prepare.R 3 | \name{branch_install_impl} 4 | \alias{branch_install_impl} 5 | \title{Checks out a source branch and install the package} 6 | \usage{ 7 | branch_install_impl( 8 | branch = "main", 9 | path_pkg = ".", 10 | install_dependencies = FALSE 11 | ) 12 | } 13 | \arguments{ 14 | \item{branch}{The name of the branch which should be installed.} 15 | 16 | \item{path_pkg}{The path to the repository to install.} 17 | 18 | \item{install_dependencies}{Passed to \code{\link[remotes:install_local]{remotes::install_local()}}. Set to 19 | \code{FALSE} can help when ran locally without internet connection.} 20 | } 21 | \value{ 22 | A character vector with library paths. 23 | } 24 | \description{ 25 | Checks out a source branch and install the package 26 | } 27 | \keyword{internal} 28 | -------------------------------------------------------------------------------- /man/branches_upsample.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{branches_upsample} 4 | \alias{branches_upsample} 5 | \title{Samples \code{branch}} 6 | \usage{ 7 | branches_upsample(branch, n = 20) 8 | } 9 | \description{ 10 | A block is a permutation of all unique elements in \code{branch}. Then, we sample 11 | \code{n} blocks. This is better than repeating one sample a certain number of 12 | times because if compute resources steadily increase, the first sample will 13 | always perform worse than the second, so the order within the blocks must be 14 | random. 15 | } 16 | \keyword{internal} 17 | -------------------------------------------------------------------------------- /man/cache_up_to_date.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/prepare.R 3 | \name{cache_up_to_date} 4 | \alias{cache_up_to_date} 5 | \alias{cache_update} 6 | \alias{cache_get} 7 | \title{Cache package sources within a session} 8 | \usage{ 9 | cache_up_to_date(branch, path_pkg) 10 | 11 | cache_update(branch, path_pkg) 12 | 13 | cache_get() 14 | } 15 | \arguments{ 16 | \item{path_pkg}{The path to the repository to install.} 17 | } 18 | \description{ 19 | This is required to make sure \code{\link[remotes:install_local]{remotes::install_local()}} installs again 20 | when source code changed. 21 | } 22 | \keyword{internal} 23 | -------------------------------------------------------------------------------- /man/figures/README-example-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lorenzwalthert/touchstone/f17ea09bc1835dfc8705135ba406fec85c25aa17/man/figures/README-example-1.png -------------------------------------------------------------------------------- /man/figures/screenshot-pr-comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lorenzwalthert/touchstone/f17ea09bc1835dfc8705135ba406fec85c25aa17/man/figures/screenshot-pr-comment.png -------------------------------------------------------------------------------- /man/figures/workflow-visualization.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lorenzwalthert/touchstone/f17ea09bc1835dfc8705135ba406fec85c25aa17/man/figures/workflow-visualization.png -------------------------------------------------------------------------------- /man/hash_pkg.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/prepare.R 3 | \name{hash_pkg} 4 | \alias{hash_pkg} 5 | \title{When did the package sources change last?} 6 | \usage{ 7 | hash_pkg(path_pkg) 8 | } 9 | \arguments{ 10 | \item{path_pkg}{The path to the repository to install.} 11 | } 12 | \description{ 13 | When did the package sources change last? 14 | } 15 | \keyword{internal} 16 | -------------------------------------------------------------------------------- /man/install_missing_deps.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/prepare.R 3 | \name{install_missing_deps} 4 | \alias{install_missing_deps} 5 | \title{Install missing BASE dependencies} 6 | \usage{ 7 | install_missing_deps(path_pkg, quiet = FALSE) 8 | } 9 | \description{ 10 | If the HEAD branch removes dependencies (compared to BASE), installing the 11 | package from the BASE branch will fail due to missing dependencies, as 12 | dependencies were installed in the GitHub Action based on the \code{DESCRIPTION} 13 | of the HEAD branch. The simplest way to ensure all required dependencies are 14 | present (including specification of remotes) is by simply installing them 15 | into the respective {touchstone} library. Prepend the local touchstone 16 | library to the library path with \code{\link[=local_touchstone_libpath]{local_touchstone_libpath()}}. 17 | } 18 | \keyword{internal} 19 | -------------------------------------------------------------------------------- /man/is_installed.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{is_installed} 4 | \alias{is_installed} 5 | \title{Check if a package is installed and unloading it} 6 | \usage{ 7 | is_installed(path_pkg = ".") 8 | } 9 | \description{ 10 | Check if a package is installed and unloading it 11 | } 12 | \keyword{internal} 13 | -------------------------------------------------------------------------------- /man/local_clean_touchstone.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/testing.R 3 | \name{local_clean_touchstone} 4 | \alias{local_clean_touchstone} 5 | \title{Clean up} 6 | \usage{ 7 | local_clean_touchstone(envir = parent.frame()) 8 | } 9 | \arguments{ 10 | \item{envir}{\verb{[environment]}\cr Attach exit handlers to this environment. 11 | Typically, this should be either the current environment or 12 | a parent frame (accessed through \code{\link[=parent.frame]{parent.frame()}}).} 13 | } 14 | \description{ 15 | Deletes \code{\link[=dir_touchstone]{dir_touchstone()}} when the local frame is destroyed. 16 | } 17 | \seealso{ 18 | Other testers: 19 | \code{\link{local_package}()} 20 | } 21 | \concept{testers} 22 | \keyword{internal} 23 | -------------------------------------------------------------------------------- /man/local_package.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/testing.R 3 | \name{local_package} 4 | \alias{local_package} 5 | \title{Create a test package} 6 | \usage{ 7 | local_package( 8 | pkg_name = fs::path_file(fs::file_temp("pkg")), 9 | branches = c("main", "devel"), 10 | r_sample = NULL, 11 | setwd = TRUE, 12 | envir = parent.frame() 13 | ) 14 | } 15 | \arguments{ 16 | \item{pkg_name}{The name of the temporary package.} 17 | 18 | \item{branches}{Branches to be created.} 19 | 20 | \item{r_sample}{Character with code to write to \code{R/sampleR.}. This is helpful 21 | to validate if the installed package corresponds to source branch for 22 | testing. If \code{NULL}, nothing is written.} 23 | 24 | \item{setwd}{Whether or not the working directory should be temporarily 25 | set to the package root.} 26 | 27 | \item{envir}{\verb{[environment]}\cr Attach exit handlers to this environment. 28 | Typically, this should be either the current environment or 29 | a parent frame (accessed through \code{\link[=parent.frame]{parent.frame()}}).} 30 | } 31 | \description{ 32 | Creates a package in a temporary directory and sets the working directory 33 | until the local frame is destroyed. 34 | } 35 | \details{ 36 | This is primarily for testing. 37 | } 38 | \seealso{ 39 | Other testers: 40 | \code{\link{local_clean_touchstone}()} 41 | } 42 | \concept{testers} 43 | \keyword{internal} 44 | -------------------------------------------------------------------------------- /man/local_touchstone_libpath.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/source.R 3 | \name{local_touchstone_libpath} 4 | \alias{local_touchstone_libpath} 5 | \title{Set Library Path} 6 | \usage{ 7 | local_touchstone_libpath(branch, env = parent.frame()) 8 | } 9 | \arguments{ 10 | \item{branch}{Git branch to use, e.g. HEAD or BASE branch.} 11 | 12 | \item{env}{Environment in which the change should be applied.} 13 | } 14 | \description{ 15 | Temporarily add a touchstone library to the path, so it can be found by 16 | \code{\link[=.libPaths]{.libPaths()}} and friends. Can be used in \link{touchstone_script} 17 | to prepare benchmarks etc. If there are touchstone libraries on the path 18 | when this function is called, they will be removed. 19 | } 20 | \seealso{ 21 | \code{\link[=run_script]{run_script()}} 22 | } 23 | \keyword{internal} 24 | -------------------------------------------------------------------------------- /man/local_without_touchstone_lib.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{local_without_touchstone_lib} 4 | \alias{local_without_touchstone_lib} 5 | \title{Temporarily remove all touchstone libraries from the path} 6 | \usage{ 7 | local_without_touchstone_lib(path_pkg = ".", envir = parent.frame()) 8 | } 9 | \arguments{ 10 | \item{path_pkg}{The path to the package that contains the touchstone library.} 11 | 12 | \item{envir}{The environment that triggers the deferred action on 13 | destruction.} 14 | } 15 | \description{ 16 | This is useful in conjunction with \code{\link[=run_script]{run_script()}}. 17 | } 18 | \details{ 19 | \itemize{ 20 | \item Add a touchstone library to the path with \code{\link[=run_script]{run_script()}} and 21 | run a script. The script hence may contain calls to libraries only 22 | installed in touchstone libraries. 23 | \item benchmark code with \code{\link[=benchmark_run]{benchmark_run()}}. At the start, remove all 24 | all touchstone libraries from path and add the touchstone library we need. 25 | } 26 | 27 | Advantages: Keep benchmarked repo in touchstone library only. 28 | } 29 | \keyword{internal} 30 | -------------------------------------------------------------------------------- /man/path_pinned_asset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{path_pinned_asset} 4 | \alias{path_pinned_asset} 5 | \title{Get path to asset} 6 | \usage{ 7 | path_pinned_asset(..., branch = branch_get_or_fail("GITHUB_HEAD_REF")) 8 | } 9 | \arguments{ 10 | \item{...}{character vectors, if any values are NA, the result will also be 11 | NA. The paths follow the recycling rules used in the tibble package, 12 | namely that only length 1 arguments are recycled.} 13 | 14 | \item{branch}{The branch the passed asset was copied from.} 15 | } 16 | \value{ 17 | The absolute path to the asset. 18 | } 19 | \description{ 20 | Get the path to a pinned asset within a \link{touchstone_script}. 21 | } 22 | \seealso{ 23 | \code{\link[=pin_assets]{pin_assets()}} 24 | } 25 | -------------------------------------------------------------------------------- /man/pin_assets.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{pin_assets} 4 | \alias{pin_assets} 5 | \title{Pin asset directory} 6 | \usage{ 7 | pin_assets( 8 | ..., 9 | branch = branch_get_or_fail("GITHUB_HEAD_REF"), 10 | overwrite = TRUE 11 | ) 12 | } 13 | \arguments{ 14 | \item{...}{Any number of directories or files, as strings, that you want to 15 | access in your \link{touchstone_script}.} 16 | 17 | \item{branch}{The branch the passed assets are copied from.} 18 | 19 | \item{overwrite}{Overwrite files if they exist. If this is \code{FALSE} and the 20 | file exists an error will be thrown.} 21 | } 22 | \value{ 23 | The asset directory invisibly. 24 | } 25 | \description{ 26 | Pin files or directories that need to be available on both branches when 27 | running the \link{touchstone_script}. During \code{\link[=benchmark_run]{benchmark_run()}} they will 28 | available via \code{\link[=path_pinned_asset]{path_pinned_asset()}}. This is only possible for assets 29 | \emph{within} the git repository. 30 | } 31 | \details{ 32 | When passing nested directories or files within nested directories 33 | the path will be copied recursively. See examples. 34 | } 35 | \examples{ 36 | \dontrun{ 37 | # In the touchstone script within the repo "/home/user/pkg" 38 | 39 | pin_assets(c("/home/user/pkg/bench", "inst/setup.R", "some/nested/dir")) 40 | 41 | source(path_pinned_asset("inst/setup.R")) 42 | load(path_pinned_asset("some/nested/dir/data.RData")) 43 | 44 | touchstone::benchmark_run( 45 | expr_before_benchmark = { 46 | !!setup 47 | source(path_pinned_asset("bench/exprs.R")) 48 | }, 49 | run_me = some_exprs(), 50 | n = 6 51 | ) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /man/pr_comment.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/analyze.R 3 | \name{pr_comment} 4 | \alias{pr_comment} 5 | \title{Modifying the PR Comment} 6 | \description{ 7 | The files \code{touchstone/header.R} and \code{touchstone/footer.R} allow you to modify 8 | the PR comment. The files will be evaluated in the context of 9 | \code{\link[=benchmark_analyze]{benchmark_analyze()}} and should return one string containing the text. 10 | You can use github markdown e.g. emojis like :tada: in the string. 11 | } 12 | \section{Header}{ 13 | 14 | Available variables for glue substitution: 15 | \itemize{ 16 | \item ci: confidence interval 17 | \item branches: BASE and HEAD branches benchmarked against each other. 18 | } 19 | } 20 | 21 | \section{Footer}{ 22 | 23 | There are no special variables available in the footer. 24 | You can access the benchmark results via \code{\link[=path_pr_comment]{path_pr_comment()}}. 25 | } 26 | 27 | \seealso{ 28 | \code{\link[base:eval]{base::eval()}} \code{\link[base:parse]{base::parse()}} 29 | } 30 | -------------------------------------------------------------------------------- /man/run_script.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/source.R 3 | \name{run_script} 4 | \alias{run_script} 5 | \title{Sources a script} 6 | \usage{ 7 | run_script( 8 | path = "touchstone/script.R", 9 | branch = branch_get_or_fail("GITHUB_HEAD_REF") 10 | ) 11 | } 12 | \arguments{ 13 | \item{path}{The script to run. It must fulfill the requirements of a 14 | \link{touchstone_script}.} 15 | 16 | \item{branch}{The branch that corresponds to the library that should be prepended 17 | to the library path when the script at \code{path} is executed, see 'Why this 18 | function?' below.} 19 | } 20 | \value{ 21 | The same as \code{\link[base:source]{base::source()}}, which inherits from \code{\link[base:withVisible]{base::withVisible()}}, i.e. 22 | a list with \code{value} and \code{visible} (invisibly). 23 | } 24 | \description{ 25 | Basically \code{\link[base:source]{base::source()}}, but prepending the library path with a 26 | touchstone library and running the script in a temp directory to avoid 27 | git operations like checking out different branches to interfere with the 28 | script execution (as running the script changes itself through git checkout). 29 | } 30 | \section{How to run this interactively?}{ 31 | 32 | You can use \code{\link[=activate]{activate()}} to setup the environment to interactively run your 33 | script, as there are some adjustments needed to mirror the Github Action 34 | environment. 35 | In a GitHub Action workflow, the environment variables \code{GITHUB_BASE_REF} and 36 | \code{GITHUB_HEAD_REF} denote the target and source branch of the pull request - 37 | and these are default arguments in \code{\link[=benchmark_run]{benchmark_run()}} (and other functions 38 | you probably want to call in your benchmarking script) to determinate the 39 | branches to use. 40 | } 41 | 42 | \section{Why this function?}{ 43 | 44 | For isolation, \{touchstone\} does not allow the benchmarked package to be 45 | installed in the global package library, but only in touchstone libraries, as 46 | asserted with \code{\link[=assert_no_global_installation]{assert_no_global_installation()}}. However, this also implies 47 | that the package is not available in the touchstone script outside of 48 | benchmark runs (i.e. outside of \code{\link[=benchmark_run]{benchmark_run()}}. We sometimes still 49 | want to call that package to prepare a benchmarking run though. To 50 | allow this, we prepend a touchstone library location that 51 | contains the installed benchmarked package for set-up tasks, and temporarily 52 | remove it during benchmarking with \code{\link[=benchmark_run]{benchmark_run()}} so only one 53 | touchstone library is on the library path at any time. 54 | } 55 | 56 | \examples{ 57 | \dontrun{ 58 | # assuming you want to compare the branch main with the branch devel 59 | if (rlang::is_installed("withr")) { 60 | withr::with_envvar( 61 | c("GITHUB_BASE_REF" = "main", "GITHUB_HEAD_REF" = "devel"), 62 | run_script("touchstone/script.R") 63 | ) 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /man/touchstone-package.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/touchstone-package.R 3 | \docType{package} 4 | \name{touchstone-package} 5 | \alias{touchstone} 6 | \alias{touchstone-package} 7 | \title{touchstone: Continuous Benchmarking with Statistical Confidence Based on 'Git' Branches} 8 | \description{ 9 | A common problem of benchmarking in continuous integration is that the computational power of the virtual machines that run the job varies over time. This package allows users to benchmark two branches of the same repo in a random sequence for better comparison. 10 | } 11 | \seealso{ 12 | Useful links: 13 | \itemize{ 14 | \item \url{https://github.com/lorenzwalthert/touchstone} 15 | \item \url{https://lorenzwalthert.github.io/touchstone} 16 | \item Report bugs at \url{https://github.com/lorenzwalthert/touchstone/issues} 17 | } 18 | 19 | } 20 | \author{ 21 | \strong{Maintainer}: Lorenz Walthert \email{lorenz.walthert@icloud.com} 22 | 23 | Authors: 24 | \itemize{ 25 | \item Jacob Wujciak-Jens \email{jacob@wujciak.de} (\href{https://orcid.org/0000-0002-7281-3989}{ORCID}) 26 | } 27 | 28 | } 29 | \keyword{internal} 30 | -------------------------------------------------------------------------------- /man/touchstone_managers.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{touchstone_managers} 4 | \alias{touchstone_managers} 5 | \alias{dir_touchstone} 6 | \alias{touchstone_clear} 7 | \alias{path_pr_comment} 8 | \title{Touchstone managers} 9 | \usage{ 10 | dir_touchstone() 11 | 12 | touchstone_clear(all = FALSE) 13 | 14 | path_pr_comment() 15 | } 16 | \arguments{ 17 | \item{all}{Whether to clear the whole touchstone directory or just the 18 | records sub directory.} 19 | } 20 | \value{ 21 | Character vector of length one with th path to the touchstone directory. 22 | 23 | The deleted paths (invisibly). 24 | 25 | Character vector of length one with the path to the pr comment. 26 | } 27 | \description{ 28 | Utilities to manage the touchstone database. 29 | } 30 | \section{Functions}{ 31 | \itemize{ 32 | \item \code{dir_touchstone()}: returns the directory where the touchstone 33 | database lives. 34 | 35 | \item \code{touchstone_clear()}: clears the touchstone database. 36 | 37 | \item \code{path_pr_comment()}: returns the path to the file containing the pr comment. 38 | 39 | }} 40 | \seealso{ 41 | \link{pr_comment} 42 | } 43 | -------------------------------------------------------------------------------- /man/touchstone_script.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/source.R 3 | \name{touchstone_script} 4 | \alias{touchstone_script} 5 | \title{The script for benchmarking} 6 | \description{ 7 | The script that contains the code which executes the benchmark. It is 8 | typically called with \code{\link[=run_script]{run_script()}}. 9 | } 10 | \section{Requirements}{ 11 | 12 | 13 | A touchstone script must: 14 | \itemize{ 15 | \item install all versions of the benchmarked repository with \code{\link[=branch_install]{branch_install()}}. 16 | \item create benchmarks with one or more calls to \code{\link[=benchmark_run]{benchmark_run()}}. 17 | \item produce the artifacts required in the GitHub workflow with 18 | \code{\link[=benchmark_analyze]{benchmark_analyze()}}. 19 | } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /man/use_touchstone.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/use.R 3 | \name{use_touchstone} 4 | \alias{use_touchstone} 5 | \title{Initiate {touchstone}} 6 | \usage{ 7 | use_touchstone( 8 | overwrite = FALSE, 9 | command = NULL, 10 | limit_to = c("OWNER", "MEMBER", "COLLABORATOR"), 11 | force_upstream = TRUE 12 | ) 13 | } 14 | \arguments{ 15 | \item{overwrite}{Overwrites files if they exist.} 16 | 17 | \item{command}{If set to \code{NULL} (the default) will run the workflow on every 18 | commit. If set to a command (e.g. \verb{/benchmark}) the benchmark will only run 19 | when triggered with a comment on the PR starting with the command.} 20 | 21 | \item{limit_to}{Roles that are allowed to trigger the benchmark workflow 22 | via comment. See details for a list of roles and their definition. 23 | Set to \code{NULL} to allow everyone to trigger a benchmark.} 24 | 25 | \item{force_upstream}{Always benchmark against the upstream base branch.} 26 | } 27 | \value{ 28 | The function is called for its side effects and returns \code{NULL} (invisibly). 29 | } 30 | \description{ 31 | This function will initialize {touchstone} in your package repository, use 32 | from root directory. 33 | } 34 | \details{ 35 | For more information see the 'Using touchstone' vignette: 36 | `vignette("touchstone", package = "touchstone") 37 | } 38 | \examples{ 39 | \dontrun{ 40 | # within your repository 41 | use_touchstone() 42 | } 43 | } 44 | \seealso{ 45 | \code{\link[=use_touchstone_workflows]{use_touchstone_workflows()}} 46 | } 47 | -------------------------------------------------------------------------------- /man/use_touchstone_workflows.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/use.R 3 | \name{use_touchstone_workflows} 4 | \alias{use_touchstone_workflows} 5 | \title{Use touchstone GitHub Actions Workflows} 6 | \usage{ 7 | use_touchstone_workflows( 8 | overwrite = FALSE, 9 | command = NULL, 10 | limit_to = c("OWNER", "MEMBER", "COLLABORATOR"), 11 | force_upstream = TRUE 12 | ) 13 | } 14 | \arguments{ 15 | \item{overwrite}{Overwrites files if they exist.} 16 | 17 | \item{command}{If set to \code{NULL} (the default) will run the workflow on every 18 | commit. If set to a command (e.g. \verb{/benchmark}) the benchmark will only run 19 | when triggered with a comment on the PR starting with the command.} 20 | 21 | \item{limit_to}{Roles that are allowed to trigger the benchmark workflow 22 | via comment. See details for a list of roles and their definition. 23 | Set to \code{NULL} to allow everyone to trigger a benchmark.} 24 | 25 | \item{force_upstream}{Always benchmark against the upstream base branch.} 26 | } 27 | \value{ 28 | The function is called for its side effects and returns \code{NULL} (invisibly). 29 | } 30 | \description{ 31 | This function will add (or update) the {touchstone} GitHub Actions workflows 32 | to your package repository. Use in the root directory of your repository. 33 | This function will be called by \code{\link[=use_touchstone]{use_touchstone()}}, you should 34 | only need to call it to update the workflows or change their parameters. 35 | } 36 | \details{ 37 | Possible roles for \code{limit_to}: 38 | \itemize{ 39 | \item \code{OWNER}: Owner of the repository, e.g. user for user/repo. 40 | \itemize{ 41 | \item It is undocumented who holds this status in an org. 42 | } 43 | \item \code{MEMBER}: Member of org for org/repo. 44 | \item \code{COLLABORATOR}: Anyone who was added as a collaborator to a repository. 45 | \item \code{CONTRIBUTOR}: Anyone who has contributed any commit to the repository. 46 | } 47 | 48 | Each user has only one role and the check does not interpolate permissions, 49 | so you have to add all roles whom you want to have permission to start the 50 | benchmark. So if you only add "COLLABORATOR" the owner will not be able to 51 | start the benchmark. 52 | 53 | GitHub will recognize additional, mostly unusual roles, see the 54 | \href{https://docs.github.com/en/rest/issues/comments}{documentation}. 55 | } 56 | -------------------------------------------------------------------------------- /tests/testthat.R: -------------------------------------------------------------------------------- 1 | library(testthat) 2 | library(touchstone) 3 | 4 | test_check("touchstone") 5 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/source.md: -------------------------------------------------------------------------------- 1 | # activate warns [plain] 2 | 3 | Code 4 | activate() 5 | 6 | --- 7 | 8 | Code 9 | activate() 10 | Condition 11 | Warning: 12 | `activate()` is meant for interactive useonly, make sure 'script.R' works as intended! 13 | 14 | --- 15 | 16 | Code 17 | activate() 18 | Output 19 | ::warning ::activate() is meant for interactive use only, make sure script.R works as intended! 20 | 21 | # activate warns [ansi] 22 | 23 | Code 24 | activate() 25 | 26 | --- 27 | 28 | Code 29 | activate() 30 | Condition 31 | Warning: 32 | `activate()` is meant for interactive useonly, make sure script.R works as intended! 33 | 34 | --- 35 | 36 | Code 37 | activate() 38 | Output 39 | ::warning ::activate() is meant for interactive use only, make sure script.R works as intended! 40 | 41 | # activate warns [unicode] 42 | 43 | Code 44 | activate() 45 | 46 | --- 47 | 48 | Code 49 | activate() 50 | Condition 51 | Warning: 52 | `activate()` is meant for interactive useonly, make sure 'script.R' works as intended! 53 | 54 | --- 55 | 56 | Code 57 | activate() 58 | Output 59 | ::warning ::activate() is meant for interactive use only, make sure script.R works as intended! 60 | 61 | # activate warns [fancy] 62 | 63 | Code 64 | activate() 65 | 66 | --- 67 | 68 | Code 69 | activate() 70 | Condition 71 | Warning: 72 | `activate()` is meant for interactive useonly, make sure script.R works as intended! 73 | 74 | --- 75 | 76 | Code 77 | activate() 78 | Output 79 | ::warning ::activate() is meant for interactive use only, make sure script.R works as intended! 80 | 81 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/use.md: -------------------------------------------------------------------------------- 1 | # can initialize with cli [plain] 2 | 3 | Code 4 | use_touchstone() 5 | Message 6 | v Populated file 'script.R' in 'touchstone/'. 7 | v Populated file 'header.R' in 'touchstone/'. 8 | v Populated file 'footer.R' in 'touchstone/'. 9 | v Populated file 'config.json' in 'touchstone/'. 10 | v Populated file '.gitignore' in 'touchstone/'. 11 | v Populated file 'touchstone-receive.yaml' in '.github/workflows/'. 12 | v Populated file 'touchstone-comment.yaml' in '.github/workflows/'. 13 | ! Could not find '.Rbuildignore' to add 'touchstone'. 14 | * Replace the mtcars sample code in `touchstone/script.R` with code from your 15 | package you want to benchmark. 16 | i You can modify the PR comment, see `?touchstone::pr_comment`. 17 | * Commit and push to GitHub to the default branch to activate the workflow, 18 | then make a pull request to trigger your first benchmark run. 19 | 20 | # can initialize with cli [ansi] 21 | 22 | Code 23 | use_touchstone() 24 | Message 25 | v Populated file script.R in touchstone/. 26 | v Populated file header.R in touchstone/. 27 | v Populated file footer.R in touchstone/. 28 | v Populated file config.json in touchstone/. 29 | v Populated file .gitignore in touchstone/. 30 | v Populated file touchstone-receive.yaml in .github/workflows/. 31 | v Populated file touchstone-comment.yaml in .github/workflows/. 32 | ! Could not find .Rbuildignore to add touchstone. 33 | * Replace the mtcars sample code in `touchstone/script.R` with code from your 34 | package you want to benchmark. 35 | i You can modify the PR comment, see `?touchstone::pr_comment`. 36 | * Commit and push to GitHub to the default branch to activate the workflow, 37 | then make a pull request to trigger your first benchmark run. 38 | 39 | # can initialize with cli [unicode] 40 | 41 | Code 42 | use_touchstone() 43 | Message 44 | ✔ Populated file 'script.R' in 'touchstone/'. 45 | ✔ Populated file 'header.R' in 'touchstone/'. 46 | ✔ Populated file 'footer.R' in 'touchstone/'. 47 | ✔ Populated file 'config.json' in 'touchstone/'. 48 | ✔ Populated file '.gitignore' in 'touchstone/'. 49 | ✔ Populated file 'touchstone-receive.yaml' in '.github/workflows/'. 50 | ✔ Populated file 'touchstone-comment.yaml' in '.github/workflows/'. 51 | ! Could not find '.Rbuildignore' to add 'touchstone'. 52 | • Replace the mtcars sample code in `touchstone/script.R` with code from your 53 | package you want to benchmark. 54 | ℹ You can modify the PR comment, see `?touchstone::pr_comment`. 55 | • Commit and push to GitHub to the default branch to activate the workflow, 56 | then make a pull request to trigger your first benchmark run. 57 | 58 | # can initialize with cli [fancy] 59 | 60 | Code 61 | use_touchstone() 62 | Message 63 | ✔ Populated file script.R in touchstone/. 64 | ✔ Populated file header.R in touchstone/. 65 | ✔ Populated file footer.R in touchstone/. 66 | ✔ Populated file config.json in touchstone/. 67 | ✔ Populated file .gitignore in touchstone/. 68 | ✔ Populated file touchstone-receive.yaml in .github/workflows/. 69 | ✔ Populated file touchstone-comment.yaml in .github/workflows/. 70 | ! Could not find .Rbuildignore to add touchstone. 71 | • Replace the mtcars sample code in `touchstone/script.R` with code from your 72 | package you want to benchmark. 73 | ℹ You can modify the PR comment, see `?touchstone::pr_comment`. 74 | • Commit and push to GitHub to the default branch to activate the workflow, 75 | then make a pull request to trigger your first benchmark run. 76 | 77 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/use/receive_all.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Benchmarks (Receive) 2 | 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.head_ref }} 5 | cancel-in-progress: true 6 | 7 | on: 8 | issue_comment: 9 | types: ['created', 'edited'] 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | prepare: 16 | runs-on: ubuntu-latest 17 | if: 18 | github.event.issue.pull_request && 19 | startsWith(github.event.comment.body, '/benchmark') && 20 | ( 21 | github.event.comment.author_association == 'OWNER' 22 | ) 23 | outputs: 24 | config: ${{ steps.read_touchstone_config.outputs.config }} 25 | steps: 26 | - name: Checkout repo 27 | uses: actions/checkout@v3 28 | with: 29 | fetch-depth: 0 30 | 31 | - id: read_touchstone_config 32 | run: | 33 | content=`cat ./touchstone/config.json` 34 | # the following lines are only required for multi line json 35 | content="${content//'%'/'%25'}" 36 | content="${content//$'\n'/'%0A'}" 37 | content="${content//$'\r'/'%0D'}" 38 | # end of optional handling for multi line json 39 | echo "::set-output name=config::$content" 40 | build: 41 | needs: prepare 42 | runs-on: ${{ matrix.config.os }} 43 | strategy: 44 | fail-fast: false 45 | matrix: 46 | config: 47 | - ${{ fromJson(needs.prepare.outputs.config) }} 48 | env: 49 | R_REMOTES_NO_ERRORS_FROM_WARNINGS: true 50 | RSPM: ${{ matrix.config.rspm }} 51 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 52 | steps: 53 | - uses: lorenzwalthert/touchstone/actions/receive@v1 54 | with: 55 | cache-version: 1 56 | benchmarking_repo: ${{ matrix.config.benchmarking_repo }} 57 | benchmarking_ref: ${{ matrix.config.benchmarking_ref }} 58 | benchmarking_path: ${{ matrix.config.benchmarking_path }} 59 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/use/receive_command.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Benchmarks (Receive) 2 | 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.head_ref }} 5 | cancel-in-progress: true 6 | 7 | on: 8 | issue_comment: 9 | types: ['created', 'edited'] 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | prepare: 16 | runs-on: ubuntu-latest 17 | if: 18 | github.event.issue.pull_request && 19 | startsWith(github.event.comment.body, '/benchmark') && 20 | ( 21 | github.event.comment.author_association == 'OWNER' || 22 | github.event.comment.author_association == 'MEMBER' || 23 | github.event.comment.author_association == 'COLLABORATOR' 24 | ) 25 | outputs: 26 | config: ${{ steps.read_touchstone_config.outputs.config }} 27 | steps: 28 | - name: Checkout repo 29 | uses: actions/checkout@v3 30 | with: 31 | fetch-depth: 0 32 | 33 | - id: read_touchstone_config 34 | run: | 35 | content=`cat ./touchstone/config.json` 36 | # the following lines are only required for multi line json 37 | content="${content//'%'/'%25'}" 38 | content="${content//$'\n'/'%0A'}" 39 | content="${content//$'\r'/'%0D'}" 40 | # end of optional handling for multi line json 41 | echo "::set-output name=config::$content" 42 | build: 43 | needs: prepare 44 | runs-on: ${{ matrix.config.os }} 45 | strategy: 46 | fail-fast: false 47 | matrix: 48 | config: 49 | - ${{ fromJson(needs.prepare.outputs.config) }} 50 | env: 51 | R_REMOTES_NO_ERRORS_FROM_WARNINGS: true 52 | RSPM: ${{ matrix.config.rspm }} 53 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 54 | steps: 55 | - uses: lorenzwalthert/touchstone/actions/receive@v1 56 | with: 57 | cache-version: 1 58 | benchmarking_repo: ${{ matrix.config.benchmarking_repo }} 59 | benchmarking_ref: ${{ matrix.config.benchmarking_ref }} 60 | benchmarking_path: ${{ matrix.config.benchmarking_path }} 61 | force_upstream: true 62 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/use/receive_default.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Benchmarks (Receive) 2 | 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.head_ref }} 5 | cancel-in-progress: true 6 | 7 | on: 8 | pull_request: 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | prepare: 15 | runs-on: ubuntu-latest 16 | if: 17 | true && 18 | ( 19 | github.event.pull_request.author_association == 'OWNER' || 20 | github.event.pull_request.author_association == 'MEMBER' || 21 | github.event.pull_request.author_association == 'COLLABORATOR' 22 | ) 23 | outputs: 24 | config: ${{ steps.read_touchstone_config.outputs.config }} 25 | steps: 26 | - name: Checkout repo 27 | uses: actions/checkout@v3 28 | with: 29 | fetch-depth: 0 30 | 31 | - id: read_touchstone_config 32 | run: | 33 | content=`cat ./touchstone/config.json` 34 | # the following lines are only required for multi line json 35 | content="${content//'%'/'%25'}" 36 | content="${content//$'\n'/'%0A'}" 37 | content="${content//$'\r'/'%0D'}" 38 | # end of optional handling for multi line json 39 | echo "::set-output name=config::$content" 40 | build: 41 | needs: prepare 42 | runs-on: ${{ matrix.config.os }} 43 | strategy: 44 | fail-fast: false 45 | matrix: 46 | config: 47 | - ${{ fromJson(needs.prepare.outputs.config) }} 48 | env: 49 | R_REMOTES_NO_ERRORS_FROM_WARNINGS: true 50 | RSPM: ${{ matrix.config.rspm }} 51 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 52 | steps: 53 | - uses: lorenzwalthert/touchstone/actions/receive@v1 54 | with: 55 | cache-version: 1 56 | benchmarking_repo: ${{ matrix.config.benchmarking_repo }} 57 | benchmarking_ref: ${{ matrix.config.benchmarking_ref }} 58 | benchmarking_path: ${{ matrix.config.benchmarking_path }} 59 | force_upstream: true 60 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/use/receive_limit.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Benchmarks (Receive) 2 | 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.head_ref }} 5 | cancel-in-progress: true 6 | 7 | on: 8 | pull_request: 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | prepare: 15 | runs-on: ubuntu-latest 16 | if: 17 | true && 18 | ( 19 | github.event.pull_request.author_association == 'OWNER' 20 | ) 21 | outputs: 22 | config: ${{ steps.read_touchstone_config.outputs.config }} 23 | steps: 24 | - name: Checkout repo 25 | uses: actions/checkout@v3 26 | with: 27 | fetch-depth: 0 28 | 29 | - id: read_touchstone_config 30 | run: | 31 | content=`cat ./touchstone/config.json` 32 | # the following lines are only required for multi line json 33 | content="${content//'%'/'%25'}" 34 | content="${content//$'\n'/'%0A'}" 35 | content="${content//$'\r'/'%0D'}" 36 | # end of optional handling for multi line json 37 | echo "::set-output name=config::$content" 38 | build: 39 | needs: prepare 40 | runs-on: ${{ matrix.config.os }} 41 | strategy: 42 | fail-fast: false 43 | matrix: 44 | config: 45 | - ${{ fromJson(needs.prepare.outputs.config) }} 46 | env: 47 | R_REMOTES_NO_ERRORS_FROM_WARNINGS: true 48 | RSPM: ${{ matrix.config.rspm }} 49 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 50 | steps: 51 | - uses: lorenzwalthert/touchstone/actions/receive@v1 52 | with: 53 | cache-version: 1 54 | benchmarking_repo: ${{ matrix.config.benchmarking_repo }} 55 | benchmarking_ref: ${{ matrix.config.benchmarking_ref }} 56 | benchmarking_path: ${{ matrix.config.benchmarking_path }} 57 | force_upstream: true 58 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/utils.md: -------------------------------------------------------------------------------- 1 | # branch can be sampled 2 | 3 | WAoAAAACAAQBAAACAwAAAAMTAAAAAgAAAA0AAAAoAAAAAQAAAAEAAAACAAAAAgAAAAMAAAAD 4 | AAAABAAAAAQAAAAFAAAABQAAAAYAAAAGAAAABwAAAAcAAAAIAAAACAAAAAkAAAAJAAAACgAA 5 | AAoAAAALAAAACwAAAAwAAAAMAAAADQAAAA0AAAAOAAAADgAAAA8AAAAPAAAAEAAAABAAAAAR 6 | AAAAEQAAABIAAAASAAAAEwAAABMAAAAUAAAAFAAAABAAAAAoAAQACQAAAARtYWluAAQACQAA 7 | AAdpc3N1ZS0zAAQACQAAAAdpc3N1ZS0zAAQACQAAAARtYWluAAQACQAAAAdpc3N1ZS0zAAQA 8 | CQAAAARtYWluAAQACQAAAAdpc3N1ZS0zAAQACQAAAARtYWluAAQACQAAAARtYWluAAQACQAA 9 | AAdpc3N1ZS0zAAQACQAAAAdpc3N1ZS0zAAQACQAAAARtYWluAAQACQAAAARtYWluAAQACQAA 10 | AAdpc3N1ZS0zAAQACQAAAAdpc3N1ZS0zAAQACQAAAARtYWluAAQACQAAAAdpc3N1ZS0zAAQA 11 | CQAAAARtYWluAAQACQAAAAdpc3N1ZS0zAAQACQAAAARtYWluAAQACQAAAAdpc3N1ZS0zAAQA 12 | CQAAAARtYWluAAQACQAAAARtYWluAAQACQAAAAdpc3N1ZS0zAAQACQAAAARtYWluAAQACQAA 13 | AAdpc3N1ZS0zAAQACQAAAARtYWluAAQACQAAAAdpc3N1ZS0zAAQACQAAAAdpc3N1ZS0zAAQA 14 | CQAAAARtYWluAAQACQAAAAdpc3N1ZS0zAAQACQAAAARtYWluAAQACQAAAARtYWluAAQACQAA 15 | AAdpc3N1ZS0zAAQACQAAAAdpc3N1ZS0zAAQACQAAAARtYWluAAQACQAAAAdpc3N1ZS0zAAQA 16 | CQAAAARtYWluAAQACQAAAAdpc3N1ZS0zAAQACQAAAARtYWluAAAEAgAAAAEABAAJAAAABWNs 17 | YXNzAAAAEAAAAAMABAAJAAAABnRibF9kZgAEAAkAAAADdGJsAAQACQAAAApkYXRhLmZyYW1l 18 | AAAEAgAAAAEABAAJAAAACXJvdy5uYW1lcwAAAA0AAAACgAAAAP///9gAAAQCAAAAAQAEAAkA 19 | AAAFbmFtZXMAAAAQAAAAAgAEAAkAAAAFYmxvY2sABAAJAAAABmJyYW5jaAAAAP4= 20 | 21 | --- 22 | 23 | WAoAAAACAAQBAAACAwAAAAMTAAAAAgAAAA0AAAA0AAAAAQAAAAEAAAABAAAAAQAAAAEAAAAB 24 | AAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAA 25 | AAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAAC 26 | AAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAA 27 | AAIAAAACAAAAAgAAAAIAAAACAAAAAgAAABAAAAA0AAQACQAAAAFlAAQACQAAAAFsAAQACQAA 28 | AAFnAAQACQAAAAFkAAQACQAAAAFoAAQACQAAAAFrAAQACQAAAAF2AAQACQAAAAFqAAQACQAA 29 | AAF0AAQACQAAAAFwAAQACQAAAAFyAAQACQAAAAFxAAQACQAAAAF6AAQACQAAAAFiAAQACQAA 30 | AAF5AAQACQAAAAFuAAQACQAAAAFvAAQACQAAAAF1AAQACQAAAAFhAAQACQAAAAFzAAQACQAA 31 | AAFmAAQACQAAAAFtAAQACQAAAAF3AAQACQAAAAF4AAQACQAAAAFpAAQACQAAAAFjAAQACQAA 32 | AAF2AAQACQAAAAFzAAQACQAAAAFmAAQACQAAAAFoAAQACQAAAAFvAAQACQAAAAFqAAQACQAA 33 | AAFrAAQACQAAAAFjAAQACQAAAAFlAAQACQAAAAF3AAQACQAAAAF1AAQACQAAAAFkAAQACQAA 34 | AAFhAAQACQAAAAF5AAQACQAAAAF4AAQACQAAAAF6AAQACQAAAAFpAAQACQAAAAFnAAQACQAA 35 | AAFiAAQACQAAAAFuAAQACQAAAAFwAAQACQAAAAFsAAQACQAAAAFyAAQACQAAAAF0AAQACQAA 36 | AAFxAAQACQAAAAFtAAAEAgAAAAEABAAJAAAABWNsYXNzAAAAEAAAAAMABAAJAAAABnRibF9k 37 | ZgAEAAkAAAADdGJsAAQACQAAAApkYXRhLmZyYW1lAAAEAgAAAAEABAAJAAAACXJvdy5uYW1l 38 | cwAAAA0AAAACgAAAAP///8wAAAQCAAAAAQAEAAkAAAAFbmFtZXMAAAAQAAAAAgAEAAkAAAAF 39 | YmxvY2sABAAJAAAABmJyYW5jaAAAAP4= 40 | 41 | # git root is found correctly [plain] 42 | 43 | Code 44 | find_git_root(no_git) 45 | Message 46 | x Could not find git repository from current working directory! 47 | i Please manually set the option "touchstone.git_root". 48 | Output 49 | NULL 50 | 51 | # git root is found correctly [ansi] 52 | 53 | Code 54 | find_git_root(no_git) 55 | Message 56 | x Could not find git repository from current working directory! 57 | i Please manually set the option "touchstone.git_root". 58 | Output 59 | NULL 60 | 61 | # git root is found correctly [unicode] 62 | 63 | Code 64 | find_git_root(no_git) 65 | Message 66 | ✖ Could not find git repository from current working directory! 67 | ℹ Please manually set the option "touchstone.git_root". 68 | Output 69 | NULL 70 | 71 | # git root is found correctly [fancy] 72 | 73 | Code 74 | find_git_root(no_git) 75 | Message 76 | ✖ Could not find git repository from current working directory! 77 | ℹ Please manually set the option "touchstone.git_root". 78 | Output 79 | NULL 80 | 81 | -------------------------------------------------------------------------------- /tests/testthat/test-analyze.R: -------------------------------------------------------------------------------- 1 | test_that("can analyze results", { 2 | skip_if(packageVersion("mockery") < "0.4.2.9000") 3 | withr::local_options(list(touchstone.skip_install = TRUE)) 4 | branches <- c("devel", "c4") 5 | path_test_pkg <- local_package(branches = branches) 6 | purrr::walk(branches, ~ benchmark_run_iteration( 7 | "", 8 | dots = list(xx1 = "Sys.sleep(runif(1, 0, 1e-5))"), 9 | n = 2, 10 | branch = .x, 11 | block = 1 12 | )) 13 | mockery::stub( 14 | benchmark_verbalize, 15 | "confint_relative_get", 16 | list(string = "[x.xx%, y.yy%]", emoji = ":rocket:"), 17 | depth = 2 18 | ) 19 | benchmark_analyze(branches) 20 | expect_match( 21 | readLines("touchstone/pr-comment/info.txt")[3], 22 | as.character(glue::glue("xx1: .*s -> .*s \\[.*%, .*%\\]")) 23 | ) 24 | expect_true(fs::file_exists("touchstone/plots/xx1.png")) 25 | }) 26 | 27 | 28 | test_that("can validate inputs before analysing", { 29 | expect_error(benchmark_analyze(branches = "just-one"), "exactly two branches") 30 | }) 31 | 32 | test_that("can analyze results", { 33 | skip_if(packageVersion("mockery") < "0.4.2.9000") 34 | withr::local_options(list(touchstone.skip_install = TRUE)) 35 | branches <- c("devel", "c4") 36 | path_test_pkg <- local_package(branches = branches) 37 | purrr::walk(branches, ~ benchmark_run_iteration( 38 | "", 39 | dots = list(xx1 = "Sys.sleep(runif(1, 0, 1e-5))"), 40 | n = 2, 41 | branch = .x, 42 | block = 1 43 | )) 44 | 45 | purrr::walk(branches, ~ benchmark_run_iteration( 46 | "", 47 | dots = list(xx2 = "Sys.sleep(runif(1, 0, 1e-5))"), 48 | n = 2, 49 | branch = .x, 50 | block = 1 51 | )) 52 | 53 | mockery::stub( 54 | benchmark_verbalize, 55 | "confint_relative_get", 56 | list(string = "[x.xx%, y.yy%]", emoji = ":rocket:"), 57 | depth = 2 58 | ) 59 | fs::file_delete(fs::path(dir_touchstone(), "records", "xx1", branches[1])) 60 | expect_warning( 61 | out <- benchmark_analyze(branches), 62 | "All benchmarks to analyse must have the two branches" 63 | ) 64 | expect_match( 65 | out[3], 66 | as.character(glue::glue("xx2: .*s -> .*s \\[.*%, .*%\\]")) 67 | ) 68 | expect_true(fs::file_exists("touchstone/plots/xx2.png")) 69 | }) 70 | 71 | test_that("missing package throws error", { 72 | skip_if(packageVersion("mockery") < "0.4.2.9000") 73 | mockery::stub(benchmark_analyze, "requireNamespace", FALSE) 74 | expect_error(benchmark_analyze(), "additional package") 75 | }) 76 | -------------------------------------------------------------------------------- /tests/testthat/test-core.R: -------------------------------------------------------------------------------- 1 | test_that("iterations can be run", { 2 | local_package() 3 | bm <- benchmark_run_iteration( 4 | expr_before_benchmark = library(testthat), 5 | dots = list(expr_to_benchmark = expect_equal(Sys.sleep(1e-3), NULL)), 6 | branch = "benchmark_run_iteration", 7 | block = 1, 8 | n = 2 9 | ) 10 | schema <- purrr::map_chr(bm, ~ class(.x)[1]) 11 | expect_equal(schema, schema_disk()) 12 | }) 13 | 14 | 15 | test_that("branches can be run", { 16 | path_test_pkg <- local_package() 17 | bm <- benchmark_run( 18 | expr_before_benchmark = library(testthat), 19 | bliblablup = expect_equal(Sys.sleep(1e-3), NULL), 20 | branches = "main", 21 | n = 2 22 | ) 23 | schema <- purrr::map_chr(bm, ~ class(.x)[1]) 24 | expect_equal(schema, schema_disk()) 25 | }) 26 | 27 | 28 | test_that("string input gives error", { 29 | path_test_pkg <- local_package() 30 | expect_error( 31 | benchmark_run( 32 | expr_before_benchmark = "library(testthat)", 33 | bliblablup = expect_equal(Sys.sleep(1e-3), NULL), 34 | branches = "main", 35 | n = 2 36 | ), 37 | "is deprecated." 38 | ) 39 | }) 40 | 41 | test_that("string input gives error", { 42 | path_test_pkg <- local_package() 43 | expect_error( 44 | benchmark_run( 45 | expr_before_benchmark = library(testthat), 46 | bliblablup = "expect_equal(Sys.sleep(1e-3), NULL)", 47 | branches = "main", 48 | n = 2 49 | ), 50 | "is deprecated." 51 | ) 52 | }) 53 | 54 | 55 | test_that("dynamic dots are supported", { 56 | local_package() 57 | x <- "cc" 58 | bm <- benchmark_run( 59 | expr_before_benchmark = {}, 60 | !!x := rlang::expr(Sys.sleep(0)), 61 | branches = "main", 62 | n = 1 63 | ) 64 | schema <- purrr::map_chr(bm, ~ class(.x)[1]) 65 | expect_equal(schema, schema_disk()) 66 | vec <- c(xzy = rlang::expr(Sys.sleep(0))) 67 | bm <- benchmark_run( 68 | expr_before_benchmark = {}, 69 | !!!vec, 70 | branches = "main", 71 | n = 1 72 | ) 73 | schema <- purrr::map_chr(bm, ~ class(.x)[1]) 74 | expect_equal(schema, schema_disk()) 75 | }) 76 | -------------------------------------------------------------------------------- /tests/testthat/test-end-to-end.R: -------------------------------------------------------------------------------- 1 | test_that("end to end run - code", { 2 | branches <- c("manila_master", "setova_feature") 3 | # feature is slower 4 | timings <- c(0.4, 0.8) %>% 5 | rlang::set_names(branches) 6 | local_package(branches = branches) 7 | for (branch in branches) { 8 | gert::git_branch_checkout(branch) 9 | code <- glue::glue( 10 | "f <- function() Sys.sleep(runif(1, {timings[branch]} - 0.05, {timings[branch]} + 0.05))" 11 | ) 12 | writeLines(code, "R/core.R") 13 | gert::git_add(fs::path_file(fs::dir_ls(all = TRUE))) 14 | gert::git_commit_all( 15 | glue::glue("setup branch {branch}.") 16 | ) 17 | } 18 | bm <- benchmark_run( 19 | expr_before_benchmark = source("R/core.R"), 20 | bliblablup = f(), 21 | branches = branches, 22 | n = 2 23 | ) 24 | out <- benchmark_read("bliblablup", branches) %>% 25 | dplyr::group_by(.data$branch) %>% 26 | dplyr::summarise(mean = mean(elapsed), sd = sd(elapsed)) 27 | # expect diff around 2 28 | diff <- max(out$mean) / min(out$mean) 29 | expect_lt(diff, 2.5) 30 | expect_gt(diff, 1.5) 31 | out <- benchmark_analyze_impl("bliblablup", branches) 32 | expect_match( 33 | out, 34 | glue::glue("bliblablup: .* -> .*\\[.*%, .*\\]") 35 | ) 36 | }) 37 | -------------------------------------------------------------------------------- /tests/testthat/test-io.R: -------------------------------------------------------------------------------- 1 | test_that("can read, write and list benchmark", { 2 | local_clean_touchstone() 3 | branch <- "hash" 4 | atomic <- bench::mark(1 + 1, iterations = 1) 5 | expect_silent( 6 | benchmark_write(atomic, name = "x1", branch = branch) 7 | ) 8 | bm <- benchmark_read(branch, name = "x1") 9 | schema <- purrr::map_chr(bm, ~ class(.x)[1]) 10 | expect_equal(schema, schema_disk()) 11 | expect_equal(unique(benchmark_ls()$name), "x1") 12 | expect_equal(benchmark_ls(name = "x1"), tibble::tibble(name = "x1", branch = "hash")) 13 | }) 14 | 15 | test_that("does fail infomatively if there is no benchmark", { 16 | local_clean_touchstone() 17 | expect_equal(benchmark_ls(), tibble::tibble(name = character(), branch = character())) 18 | }) 19 | 20 | 21 | 22 | test_that("fails on corrupt benchmark", { 23 | local_clean_touchstone() 24 | branch <- "malformed" 25 | multiple <- bench::mark(1 + 1, iterations = 5) 26 | expect_error( 27 | benchmark_write(multiple, branch = branch), 28 | "only supports" 29 | ) 30 | }) 31 | -------------------------------------------------------------------------------- /tests/testthat/test-prepare.R: -------------------------------------------------------------------------------- 1 | test_that("can install in isolated repos", { 2 | name_tmp_pkg <- "bli33" 3 | local_package(name_tmp_pkg, r_sample = "x <- 3") 4 | lib_path1 <- branch_install("devel") 5 | expect_equal( 6 | withr::with_libpaths(lib_path1, bli33:::x), 3 7 | ) 8 | # with root != "." 9 | name_tmp_pkg <- "bli44" 10 | local_package(name_tmp_pkg, r_sample = "x <- 55") 11 | expect_error( 12 | withr::with_libpaths(lib_path1, bli44:::x), 13 | "[Tt]here is no package" 14 | ) 15 | lib_path2 <- branch_install("devel") 16 | expect_equal( 17 | withr::with_libpaths(lib_path2, bli44:::x), 55 18 | ) 19 | }) 20 | 21 | test_that("cache works", { 22 | ref <- "devel" 23 | name_tmp_pkg <- "bli44" 24 | path_pkg <- local_package(name_tmp_pkg, r_sample = "x <- 55") 25 | 26 | expect_equal(nrow(cache_get()), 0) 27 | expect_false(cache_up_to_date(ref, path_pkg)) 28 | cache_update(ref, path_pkg) 29 | expect_equal(nrow(cache_get()), 1) 30 | writeLines(c("x <- 55"), "R/sample.R") 31 | expect_true(cache_up_to_date(ref, path_pkg)) 32 | writeLines(c("22"), "R/sample.R") 33 | expect_false(cache_up_to_date(ref, path_pkg)) 34 | expect_false(cache_up_to_date(ref, path_pkg)) 35 | cache_update(ref, path_pkg) 36 | expect_true(cache_up_to_date(ref, path_pkg)) 37 | 38 | # new ref 39 | ref <- "m2" 40 | expect_equal(nrow(cache_get()), 1) 41 | # prepare for case that remotes would ever have global cache across libraries 42 | # (currently not the case) and could think "version has not changed, just copying" 43 | expect_false(cache_up_to_date(ref, path_pkg)) 44 | cache_update(ref, path_pkg) 45 | expect_equal(nrow(cache_get()), 2) 46 | cache_update(ref, path_pkg) 47 | expect_true(cache_up_to_date(ref, path_pkg)) 48 | 49 | # new root 50 | cache <- cache_get() 51 | path_pkg2 <- local_package(name_tmp_pkg, r_sample = "c") 52 | options("touchstone.hash_source_package" = cache) 53 | expect_equal(nrow(cache_get()), 2) 54 | expect_false(cache_up_to_date(ref, path_pkg2)) 55 | cache_update(ref, path_pkg2) 56 | expect_true(cache_up_to_date(ref, path_pkg2)) 57 | }) 58 | -------------------------------------------------------------------------------- /tests/testthat/test-source.R: -------------------------------------------------------------------------------- 1 | test_that("can call package in script", { 2 | branches <- c("main", "devel") 3 | pkg_name <- "b32jk" 4 | path_test_pkg <- local_package(pkg_name, branches = branches) 5 | # install branches, so branches[1] will be available outside benchmark_run, 6 | # not modifying libpath permanently 7 | path_touchstone <- path_touchstone_script() 8 | withr::local_options(touchstone.git_root = path_test_pkg) 9 | fs::dir_create(fs::path_dir(path_touchstone)) 10 | branches_dput <- capture.output(dput(branches)) 11 | path_wordlist <- fs::path(path_test_pkg, "inst", "WORDLIST") 12 | ensure_dir(fs::path_dir(path_wordlist)) 13 | writeLines("a\n\nc", path_wordlist) 14 | 15 | no_assets <- glue::glue( 16 | "branch_install({branches_dput}, install_dependencies = FALSE)", 17 | "library({pkg_name})", # can call package 18 | "touchstone::benchmark_run(", 19 | " branches = {branches_dput}, x = 2,", 20 | " n = 1", 21 | ")", 22 | .sep = "\n" 23 | ) 24 | 25 | with_assets <- glue::glue( 26 | "branch_install({branches_dput}, install_dependencies = FALSE)", 27 | "library({pkg_name})", # can call package 28 | "path <- 'inst/WORDLIST'", 29 | "touchstone::pin_assets(path, branch = 'main')", 30 | "touchstone::benchmark_run(", 31 | " expr_before_benchmark = readLines(touchstone::path_pinned_asset(!! path, branch = 'main')),", 32 | " branches = {branches_dput}, x = 2,", 33 | " n = 1", 34 | ")", 35 | .sep = "\n" 36 | ) 37 | 38 | writeLines(no_assets, path_touchstone) 39 | withr::local_envvar(GITHUB_BASE_REF = "") 40 | expect_error( 41 | run_script(path_touchstone, branch = branches[[2]]), 42 | ", you must set the environment.*variable.*which branches you want" 43 | ) 44 | 45 | withr::local_envvar(list(GITHUB_BASE_REF = "main", GITHUB_HEAD_REF = "devel")) 46 | expect_error(run_script(path_touchstone, branch = branches[[2]]), NA) 47 | writeLines(with_assets, path_touchstone) 48 | expect_error(run_script(path_touchstone), NA) 49 | }) 50 | 51 | 52 | cli::test_that_cli("activate warns", { 53 | branches <- c("main", "devel") 54 | pkg_name <- "b42jk" 55 | path_test_pkg <- local_package(pkg_name, branches = branches) 56 | local_git_checkout("devel") 57 | 58 | rlang::with_interactive( 59 | { 60 | expect_snapshot(activate()) 61 | }, 62 | TRUE 63 | ) 64 | 65 | rlang::with_interactive( 66 | { 67 | withr::local_envvar(GITHUB_ACTIONS = FALSE) 68 | expect_snapshot(activate()) 69 | }, 70 | FALSE 71 | ) 72 | 73 | rlang::with_interactive( 74 | { 75 | withr::local_envvar(GITHUB_ACTIONS = TRUE) 76 | expect_snapshot(activate()) 77 | }, 78 | FALSE 79 | ) 80 | }) 81 | -------------------------------------------------------------------------------- /tests/testthat/test-use.R: -------------------------------------------------------------------------------- 1 | cli::test_that_cli("can initialize with cli", { 2 | local_package() 3 | expect_snapshot(use_touchstone()) 4 | expect_match(conditionMessage(capture_warning(use_touchstone())), "already exists") 5 | }) 6 | 7 | test_that("Workflow template is modified correctly", { 8 | withr::with_tempdir({ 9 | # Without the temp dir testthat creates a folder the clashes with the 10 | # workflow file. 11 | 12 | file <- fs::dir_create(fs::path(".github", "workflows")) 13 | file <- fs::path(file, "touchstone-receive.yaml") 14 | 15 | use_touchstone_workflows(overwrite = TRUE) 16 | expect_snapshot_file(file, "receive_default.yml") 17 | 18 | use_touchstone_workflows(overwrite = TRUE, command = "/benchmark") 19 | expect_snapshot_file(file, "receive_command.yml") 20 | 21 | use_touchstone_workflows(overwrite = TRUE, limit_to = "OWNER") 22 | expect_snapshot_file(file, "receive_limit.yml") 23 | 24 | use_touchstone_workflows(overwrite = TRUE, command = "/benchmark", limit_to = "OWNER", force_upstream = FALSE) 25 | expect_snapshot_file(file, "receive_all.yml") 26 | }) 27 | }) 28 | -------------------------------------------------------------------------------- /tests/testthat/test-utils.R: -------------------------------------------------------------------------------- 1 | test_that("branch can be sampled", { 2 | withr::with_seed( 3 | 3, 4 | expect_snapshot_value(branches_upsample(c("main", "issue-3")), style = "serialize") 5 | ) 6 | withr::with_seed( 7 | 3, 8 | expect_snapshot_value(branches_upsample(letters, n = 2), style = "serialize") 9 | ) 10 | }) 11 | 12 | test_that("touchstone dir can be removed", { 13 | local_clean_touchstone() 14 | fs::dir_create(dir_touchstone()) 15 | touchstone_clear(all = TRUE) 16 | expect_false(fs::dir_exists(dir_touchstone())) 17 | }) 18 | 19 | test_that("can checkout locally", { 20 | local_package() 21 | old_branch <- gert::git_branch() 22 | new_branch <- "ewjlkj" 23 | gert::git_branch_create(new_branch, checkout = FALSE) 24 | gert::git_branch_list() 25 | test_f <- function(new_branch) { 26 | local_git_checkout(new_branch) 27 | expect_equal(gert::git_branch(), new_branch) 28 | } 29 | test_f(new_branch = new_branch) 30 | expect_equal(gert::git_branch(), old_branch) 31 | }) 32 | 33 | test_that("can remove touchstone libpaths", { 34 | branches <- c("devel", "main") 35 | if (is_windows()) { 36 | # cannot have touchstone library in temp, as lib path comparison becomes 37 | # unfeasible due to short/name notation 38 | withr::local_options(dir_touchstone = fs::path_file(fs::file_temp())) 39 | } 40 | path_pkg <- local_package(setwd = !is_windows()) 41 | new_libpaths <- branch_install(branches, path_pkg, install_dependencies = FALSE) 42 | 43 | withr::local_libpaths(new_libpaths) 44 | 45 | expect_equal(.libPaths(), new_libpaths) 46 | local_without_touchstone_lib(path_pkg) 47 | after_removal <- setdiff( 48 | new_libpaths, 49 | as.character(fs::path_abs(libpath_touchstone(branches))) 50 | ) 51 | 52 | expect_equal(after_removal, .libPaths()) 53 | }) 54 | 55 | 56 | test_that("Can abort with missing branches for benchmark run", { 57 | withr::local_envvar(GITHUB_BASE_REF = NA, GITHUB_HEAD_REF = NA) 58 | mockery::stub( 59 | benchmark_run, "force", 60 | function(...) cli::cli_abort("12321") 61 | ) 62 | withr::local_envvar(list( 63 | GITHUB_HEAD_REF = "feature1", 64 | GITHUB_BASE_REF = "mastero" 65 | )) 66 | 67 | expect_error( 68 | benchmark_run(x1 = print("hi")), 69 | "12321" 70 | ) 71 | }) 72 | 73 | 74 | test_that("Can abort with missing branches for benchmark run", { 75 | withr::local_envvar(GITHUB_BASE_REF = NA, GITHUB_HEAD_REF = NA) 76 | match <- "^If you don't specify" 77 | expect_error(branch_get_or_fail("SOME_REF"), match) 78 | expect_error( 79 | benchmark_run(x1 = print("hi")), 80 | match 81 | ) 82 | expect_error( 83 | benchmark_analyze_impl("sume2"), 84 | match 85 | ) 86 | 87 | expect_error( 88 | branch_install(), 89 | match 90 | ) 91 | }) 92 | 93 | test_that("assets work on HEAD", { 94 | mockery::stub(pin_assets, "local_git_checkout", TRUE) 95 | dirs <- c(fs::path_temp("test_pkg", "R"), fs::path_temp("test_pkg", "bench")) 96 | files <- c(fs::path_temp("test_pkg", "data.R"), fs::path_temp("test_pkg", "utils.R")) 97 | 98 | temp_dir <- fs::path_temp() 99 | fs::dir_create(dirs) 100 | writeLines("test", fs::path(dirs[1], "sample.R")) 101 | fs::file_create(files) 102 | dirs <- fs::path_real(dirs) 103 | files <- fs::path_real(files) 104 | withr::local_envvar(list( 105 | GITHUB_BASE_REF = "main", 106 | GITHUB_HEAD_REF = "devel" 107 | )) 108 | 109 | withr::with_options( 110 | list( 111 | touchstone.dir_assets_devel = NULL, 112 | touchstone.git_root = fs::path_real(fs::path_temp("test_pkg")), 113 | usethis.quiet = TRUE 114 | ), 115 | { 116 | expect_error(pin_assets("something"), "Temporary directory for branch") 117 | expect_error(path_pinned_asset("something"), "Temporary directory ") 118 | } 119 | ) 120 | 121 | withr::with_options(list( 122 | touchstone.dir_assets_devel = temp_dir, 123 | touchstone.git_root = fs::path_real(fs::path_temp("test_pkg")), 124 | usethis.quiet = TRUE 125 | ), { 126 | expect_warning(pin_assets("something", dirs[[1]]), "could not be found") 127 | expect_error(suppressWarnings(pin_assets("something")), "No valid") 128 | expect_equal(pin_assets(!!!dirs), temp_dir) 129 | expect_equal( 130 | readLines( 131 | fs::path( 132 | path_pinned_asset( 133 | fs::path_rel(dirs[1], getOption("touchstone.git_root")) 134 | ), 135 | "sample.R" 136 | ) 137 | ), 138 | "test" 139 | ) 140 | expect_equal(pin_assets(!!!files), temp_dir) 141 | expect_true(fs::is_dir(fs::path_join(c(temp_dir, "R")))) 142 | expect_true(fs::is_file(fs::path_join(c(temp_dir, "data.R")))) 143 | 144 | expect_error(path_pinned_asset("something"), "not pinned at") 145 | expect_equal(path_pinned_asset("R"), fs::path(temp_dir, "R")) 146 | expect_equal(path_pinned_asset("data.R"), fs::path(temp_dir, "data.R")) 147 | }) 148 | }) 149 | 150 | test_that("assets work HEAD and BASE", { 151 | branches <- c("rc-1.0", "feat") 152 | local_asset_dir(!!!branches) 153 | git_root <- local_package(branches = branches) 154 | dirs <- c("R", "bench") %>% rlang::set_names(branches) 155 | files <- c("data.Rdata", "utils.R") %>% rlang::set_names(branches) 156 | withr::local_options(list( 157 | touchstone.git_root = fs::path_real(git_root) 158 | )) 159 | 160 | for (branch in branches) { 161 | gert::git_branch_checkout(branch) 162 | fs::dir_create(dirs[branch]) 163 | fs::file_create(files[branch]) 164 | } 165 | 166 | withr::local_envvar(list( 167 | GITHUB_BASE_REF = branches[[1]], 168 | GITHUB_HEAD_REF = branches[[2]] 169 | )) 170 | 171 | for (branch in branches) { 172 | pin_assets(dirs[branch], files[branch], branch = branch) 173 | } 174 | 175 | expect_true( 176 | fs::file_exists( 177 | path_pinned_asset("data.Rdata", branch = branches[[1]]) 178 | )[[1]] 179 | ) 180 | expect_true( 181 | fs::file_exists( 182 | path_pinned_asset("utils.R", branch = branches[[2]]) 183 | )[[1]] 184 | ) 185 | }) 186 | 187 | test_that("asset paths are fetched correctly", { 188 | withr::local_options(list( 189 | touchstone.dir_assets_devel = "asset/dir", 190 | touchstone.dir_assets_main = NULL 191 | )) 192 | 193 | withr::local_envvar(list( 194 | GITHUB_BASE_REF = "main", 195 | GITHUB_HEAD_REF = "devel" 196 | )) 197 | 198 | expect_error(get_asset_dir("main"), "directory for branch") 199 | expect_equal(get_asset_dir("devel"), "asset/dir") 200 | }) 201 | 202 | cli::test_that_cli("git root is found correctly", { 203 | no_git <- fs::path_temp("no-git") 204 | with_git <- fs::path_temp("with-git") 205 | deeper_git <- fs::path_temp("with-git", "deep", "deeper") 206 | fs::dir_create(c(no_git, with_git, deeper_git)) 207 | 208 | no_git <- fs::path_real(no_git) 209 | with_git <- fs::path_real(with_git) 210 | deeper_git <- fs::path_real(deeper_git) 211 | 212 | withr::with_dir(with_git, { 213 | gert::git_init() 214 | }) 215 | 216 | expect_snapshot(find_git_root(no_git)) 217 | expect_equal(find_git_root(with_git), as.character(with_git)) 218 | expect_equal(find_git_root(deeper_git), as.character(with_git)) 219 | }) 220 | 221 | test_that("envvar_true works", { 222 | withr::local_envvar( 223 | NOT_BOOL = 23, 224 | BOOL_T = TRUE, 225 | BOOL_t = "true" 226 | ) 227 | 228 | expect_error(envvar_true(23), "is not TRUE") 229 | expect_true(envvar_true("BOOL_T")) 230 | expect_true(envvar_true("BOOL_t")) 231 | expect_false(envvar_true("NOT_BOOL")) 232 | }) 233 | 234 | test_that("can assert no global installation", { 235 | local_package() 236 | mockery::stub(assert_no_global_installation, "is_installed", list(installed = TRUE, name = "pkg")) 237 | expect_error(assert_no_global_installation(), "can be found on a non-touchstone library path.") 238 | }) 239 | -------------------------------------------------------------------------------- /touchstone.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: No 4 | SaveWorkspace: No 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | 15 | AutoAppendNewline: Yes 16 | StripTrailingWhitespace: Yes 17 | LineEndingConversion: Posix 18 | 19 | BuildType: Package 20 | PackageUseDevtools: Yes 21 | PackageInstallArgs: --no-multiarch --with-keep.source 22 | PackageRoxygenize: rd,collate,namespace 23 | -------------------------------------------------------------------------------- /vignettes/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | *.R 3 | -------------------------------------------------------------------------------- /vignettes/inference.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Inference" 3 | output: rmarkdown::html_vignette 4 | vignette: > 5 | %\VignetteIndexEntry{Inference} 6 | %\VignetteEngine{knitr::rmarkdown} 7 | %\VignetteEncoding{UTF-8} 8 | --- 9 | 10 | ```{r, include = FALSE} 11 | knitr::opts_chunk$set( 12 | collapse = TRUE, 13 | comment = "#>", 14 | eval = FALSE 15 | ) 16 | ``` 17 | 18 | ## Interpretation 19 | 20 | We first explain the output of the comment in the PR: 21 | 22 | ![](../man/figures/screenshot-pr-comment.png) 23 | 24 | Every bullet point corresponds to one expression you benchmarked in your 25 | touchstone script in `touchstone/script.R` with `benchmark_run()`. The 26 | contents of the former are in this (simplified) example: 27 | 28 | ```{r} 29 | library(touchstone) 30 | branches_install() 31 | benchmark_run( 32 | expr_before_benchmark = c("library(styler)", "cache_deactivate()"), 33 | without_cache = 'styler::style_pkg("touchstone/sources/here")', 34 | n = 30 35 | ) 36 | 37 | benchmark_run( 38 | expr_before_benchmark = c("library(styler)", "cache_activate()"), 39 | cache_applying = 'styler::style_pkg("touchstone/sources/here")', 40 | n = 30 41 | ) 42 | 43 | benchmark_run( 44 | expr_before_benchmark = c("library(styler)", "cache_deactivate()"), 45 | cache_recording = c( 46 | "gert::git_reset_hard(repo = 'touchstone/sources/here')", 47 | 'styler::style_pkg("touchstone/sources/here")' 48 | ), 49 | n = 30 50 | ) 51 | 52 | benchmarks_analyze() 53 | ``` 54 | 55 | * You can see that the expression `cache_applying` took on average 0.1 seconds 56 | on both branches. 57 | 58 | * And if we merge this pull request, `cache_recording` and `without_cache` will 59 | be approximately 0.01 quicker than before the merge. 60 | 61 | Imagine we only ran all expressions once per branch. Then, the differences could 62 | be by chance. To avoid that, we run it `n` times and compute a confidence 63 | interval, which tells us how certain we can be about our estimated differences. 64 | A 95% confidence interval tells us that if we were to repeat the benchmarking 65 | experiment 100 times, our estimated speed change would be in the interval 95 out 66 | of 100 times. It comes from a simple ANOVA model where we regress the elapsed 67 | time on the branch from which the result comes. Like this: 68 | 69 | ```{r} 70 | fit <- stats::aov(elapsed ~ factor(branches), data = timings) 71 | confint(fit, ...) 72 | ``` 73 | 74 | We could in addition also control for `block`, but as we only have two 75 | observations per block, this required a lot of parameter estimates and would take 76 | away statistical power from our parameter of interest. Our estimates are also 77 | unbiased without it. 78 | 79 | 80 | Since changes in percent are more relevant than absolute changes, the confidence 81 | interval is reported relative to the target branch of the pull request. The 82 | measured difference is statistically significantly different from zero if the 83 | confidence interval does not overlap with 0. In the screenshot above, this is 84 | not the case for any benchmarked expression, as they all range from some 85 | negative to some positive number. If you increase `n`, you can estimate the 86 | speed implications of a pull request more precisely, meaning you'll get a more 87 | narrow confidence interval which more likely does not cover zero. You'll then 88 | easily reach statistical significance (and the CI run will take longer), but do 89 | you really care if your code gets 0.000001% slower? Probably not. That's why you 90 | also need to look at the range of the confidence interval. 91 | 92 | ## Sampling 93 | 94 | We sample both branches in `n` blocks. This gives us a more efficient estimate of 95 | the speed difference than the completely at random, because completely at random 96 | can result in or close to one of the following scenarios assuming there is an 97 | upwards trend in available compute resources over the whole period: 98 | 99 | * We run the PR branch `n` times before we run the target branch, the PR branch 100 | is at disadvantage. 101 | 102 | * If we always switch branches, the first branch is at disadvantage. 103 | 104 | The opposite effect occurs when compute power steadily decreases. So we sample 105 | randomly within blocks: 106 | 107 | ```{r, eval = TRUE} 108 | touchstone:::branches_upsample(c("main", "feature")) 109 | ``` 110 | -------------------------------------------------------------------------------- /vignettes/touchstone.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Using touchstone" 3 | output: rmarkdown::html_vignette 4 | vignette: > 5 | %\VignetteIndexEntry{Using touchstone} 6 | %\VignetteEngine{knitr::rmarkdown} 7 | %\VignetteEncoding{UTF-8} 8 | --- 9 | 10 | ```{r, include = FALSE} 11 | knitr::opts_chunk$set( 12 | collapse = TRUE, 13 | comment = "#>", 14 | eval = FALSE 15 | ) 16 | ``` 17 | 18 | # Initialization 19 | 20 | Start by initializing {touchstone} in your package repository with: 21 | 22 | ```{r, eval = FALSE} 23 | touchstone::use_touchstone() 24 | ``` 25 | 26 | This will: 27 | 28 | - create a `touchstone` directory in the repository root with: 29 | 30 | * `config.json` that defines how to run your benchmark. In particular, you can 31 | define a `benchmarking_repo`, that is, a repository you need to run your bench mark 32 | (use `""` if you don't need an additional git repository checked out for your 33 | benchmark). 34 | This repository will be cloned into `benchmarking_path`. For example to benchmark 35 | {styler}, we format the {here} package with `style_pkg()` that is not part 36 | of the {styler} repository, but with the below config, will be located at 37 | `touchstone/sources/here` during benchmarking. The code you want to 38 | benchmark comes from the benchmarked repository, which in our case is the root 39 | from where you call `use_touchstone()` and hence does not need to be defined 40 | explicitly. 41 | ```json 42 | { 43 | "benchmarking_repo": "lorenzwalthert/here", 44 | "benchmarking_ref": "ca9c8e69c727def88d8ba1c8b85b0e0bcea87b3f", 45 | "benchmarking_path": "touchstone/sources/here", 46 | "os": "ubuntu-18.04", 47 | "r": "4.0.0", 48 | "rspm": "https://packagemanager.rstudio.com/all/__linux__/bionic/291" 49 | } 50 | ``` 51 | * `script.R`, the script that runs the benchmark, also called the `touchstone_script`. 52 | * `header.R`, the script containing the default PR comment header, see `?touchstone::pr_comment`. 53 | * `footer.R`, the script containing the default PR comment footer, see `?touchstone::pr_comment`. 54 | 55 | - add the `touchstone` directory to `.Rbuildignore`. 56 | - write the workflow files^[Note that these files must be committed to the default branch before 57 | {touchstone} continuous benchmarking will be triggered for new PRs.] you need to 58 | invoke touchstone in new pull requests into `.github/workflows/`. 59 | 60 | Now you just need to modify the touchstone script `touchstone/script.R` to run 61 | the benchmarks you are interested in. 62 | 63 | # Workflow overview 64 | 65 | While you eventually want to run your benchmarks on GitHub Actions for every 66 | pull request, it is handy to develop interactively and locally first to avoid 67 | long feedback cycles with trivial errors. The 68 | below diagram shows these two different workflows and what they entail. 69 | 70 | 71 | ```{r, eval = TRUE, include = TRUE, echo = FALSE, out.width="100%"} 72 | knitr::include_graphics("../man/figures/workflow-visualization.png") 73 | ``` 74 | 75 | 76 | Note that there are two GitHub Action workflow files in `.github/workflows/` due 77 | to [security reasons](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/). 78 | 79 | In the remainder, we'll explain how to adapt the touchstone script, which is the 80 | most important script you need to customize. 81 | 82 | ## Adapting the touchstone script 83 | 84 | The file consists of three parts, two of which you will need to modify to fit your needs. Within the GitHub Action {touchstone} will always run the `touchstone/script.R` from the `HEAD` branch, so you can modify or extend the benchmarks if needed to fit your current branch. Please beware that a few 85 | functions such as `branches_install()`, `pin_assets` or `benchmark_run()` will 86 | perform local git checkouts and therefore the git status should be clean and no 87 | other processes should run in the directory. 88 | 89 | 90 | The touchstone script will be run with `run_script()` on GitHub Actions with the 91 | required environment variables `GITHUB_HEAD_REF` and `GITHUB_BASE_REF` set for 92 | various reasons explained in the help file. If you want to interactively work 93 | on your touchstone script, i.e. running it line by line, you must activate 94 | touchstone first with `activate()`, which sets environment variables, R options 95 | and library paths. 96 | 97 | ### Setup 98 | 99 | We first install the different package versions in separate libraries with `branches_install()`, this is mandatory for any `script.R`. If you want to access some directory or file which only exists on one of the branches you can use `pin_assets(..., ref = "your-branch-name")` to make them available across branches. You can of course also run arbitrary code to prepare for the benchmarks. 100 | 101 | ```{r} 102 | touchstone::branches_install() # installs branches to benchmark 103 | touchstone::pin_assets("data/all.Rdata", "inst/scripts") # pin files and directories 104 | 105 | load(path_pinned_asset("all.Rdata")) # perform other setup you need for the benchmarks 106 | source(path_pinned_asset("scripts/prepare.R")) 107 | data_clean <- prepare_data(data_all) 108 | ``` 109 | 110 | ### Benchmarks 111 | 112 | Run any number of benchmarks, with one named benchmark per call of `benchmark_run()`. As benchmarks are run in a subprocess, setup (like setting a seed) needs to be passed with `expr_before_benchmark`. You can pass a single function call or a longer block of code wrapped in "{ }". The expressions are captured with `rlang` so you can use [quasiquotation](https://rlang.r-lib.org/reference/quasiquotation.html). 113 | 114 | ```{r} 115 | # benchmark a function call from your package 116 | touchstone::benchmark_run( 117 | random_test = yourpkg::fun(data_clean), 118 | n = 2 119 | ) 120 | 121 | # Optionally run setup code 122 | touchstone::benchmark_run( 123 | expr_before_benchmark = { 124 | library(yourpkg) 125 | set.seed(42) 126 | }, 127 | test_with_seed = fun(data_clean), 128 | n = 2 129 | ) 130 | ``` 131 | 132 | ### Analyze 133 | 134 | This part is just one call to `benchmarks_analyze()`which analyses the results and prepares the PR comment. 135 | 136 | ```{r} 137 | # create artifacts used downstream in the GitHub Action 138 | touchstone::benchmarks_analyze() 139 | ``` 140 | 141 | 142 | ## Running the script 143 | 144 | After you have committed and pushed the workflow files to your default branch 145 | the benchmarks will be run on new pull requests and on each commit while that 146 | pull request is open. 147 | 148 | Note that various functions such as `branches_install()` `pin_assets()` and 149 | `benchmark_run()` will check out different branches 150 | and should therefore be the only process running in the directory and with a clean 151 | git status. 152 | 153 | 154 | ## The Results 155 | 156 | When the GitHub Action successfully completes, the main result you will see is the comment posted on the pull request, it will look something like this: 157 | 158 | ![](../man/figures/screenshot-pr-comment.png) 159 | 160 | You can change the header and footer, see `?touchstone::pr_comment`. The main body is the list of benchmarks and their results. First the mean elapsed time of all iterations of that benchmark (BASE $\rightarrow$ HEAD) and a 95% confidence interval. 161 | 162 | To make it easier to spot benchmarks that need closer attention, we prefix the benchmarks with emojis indicating if there was a statistically significant change (see [inference](https://lorenzwalthert.github.io/touchstone/articles/inference.html)) in the timing or not. In this example the HEAD version of `func1` is significantly faster, `func3` is slower than BASE while there is no significant change for `func2`. 163 | 164 | The GitHub Action will also upload this text and plots of the timings as artifacts. 165 | --------------------------------------------------------------------------------