├── .Rbuildignore ├── .Rinstignore ├── .github ├── .gitignore ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ └── feature_request.md └── workflows │ ├── R-CMD-check.yaml │ ├── future_tests.yaml │ └── test-coverage.yaml ├── .gitignore ├── .make └── Makefile ├── CONDUCT.md ├── CONTRIBUTING.md ├── DESCRIPTION ├── Makefile ├── NAMESPACE ├── NEWS.md ├── OVERVIEW.md ├── R ├── 000.import.R ├── 001.import_future_functions.R ├── BatchtoolsCustomFuture-class.R ├── BatchtoolsFuture-class.R ├── BatchtoolsFutureError.R ├── BatchtoolsMultiprocessFuture-class.R ├── BatchtoolsSSHFuture-class.R ├── BatchtoolsSSHRegistry.R ├── BatchtoolsUniprocessFuture-class.R ├── batchtools_bash.R ├── batchtools_custom.R ├── batchtools_interactive.R ├── batchtools_local.R ├── batchtools_multicore.R ├── batchtools_ssh.R ├── batchtools_template.R ├── find_template_file.R ├── future.batchtools-package.R ├── future_cache_path.R ├── nbrOfWorkers.R ├── options.R ├── resources_OP.R ├── temp_registry.R ├── utils,conditions.R ├── utils-debug.R ├── utils.R ├── waitForWorker.R ├── with_stealth_rng.R └── zzz.R ├── README.md ├── cran-comments.md ├── demo ├── 00Index └── mandelbrot.R ├── incl ├── batchtools_custom.R ├── batchtools_local.R └── future.batchtools.R ├── inst ├── CITATION ├── WORDLIST ├── templates-for-R_CMD_check │ ├── batchtools.conf.R │ ├── batchtools.sge.tmpl │ ├── batchtools.torque.tmpl │ └── setup.R └── templates │ ├── bash.tmpl │ ├── sge.tmpl │ ├── slurm.tmpl │ └── torque.tmpl ├── man ├── BatchtoolsFuture.Rd ├── BatchtoolsFutureError.Rd ├── batchtools_custom.Rd ├── batchtools_local.Rd ├── batchtools_multicore.Rd ├── batchtools_ssh.Rd ├── batchtools_template.Rd ├── delete.BatchtoolsFuture.Rd ├── future.batchtools.Rd ├── future.batchtools.options.Rd ├── grapes-resources-grapes.Rd ├── loggedOutput.Rd ├── nbrOfWorkers.batchtools.Rd └── print.BatchtoolsFuture.Rd ├── pkgdown ├── _pkgdown.yml ├── _pkgdown.yml.rsp └── favicon │ ├── apple-touch-icon-120x120.png │ ├── apple-touch-icon-60x60.png │ ├── apple-touch-icon-76x76.png │ ├── apple-touch-icon.png │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ └── favicon.ico ├── revdep ├── README.md ├── cran.md ├── failures.md ├── problems.md ├── revdepcheck.Renviron ├── run.R └── run.sge ├── tests ├── BatchtoolsFuture,gc.R ├── BatchtoolsFuture.R ├── BatchtoolsFutureError.R ├── batchtools_custom.R ├── batchtools_hpc.R ├── batchtools_interactive.R ├── batchtools_local.R ├── batchtools_multicore.R ├── batchtools_ssh.R ├── batchtools_template.R ├── demo.R ├── dotdotdot.R ├── future,labels.R ├── future,lazy.R ├── globals,formulas.R ├── globals,manual.R ├── globals,subassignment.R ├── globals,tricky.R ├── incl │ ├── end.R │ ├── start,load-only.R │ └── start.R ├── nbrOfWorkers.R ├── plan.R ├── resources_OP.R ├── rng.R ├── stdout.R ├── utils.R ├── zzz,future_lapply.R └── zzz.onUnload.R └── vignettes └── future.batchtools.md.rsp /.Rbuildignore: -------------------------------------------------------------------------------- 1 | #---------------------------- 2 | # Git and SVN related 3 | #---------------------------- 4 | ^.svn 5 | ^.git 6 | ^.github 7 | ^.make 8 | ^.local 9 | ^INSTALL[.]md$ 10 | ^OVERVIEW[.]md$ 11 | ^README[.]md$ 12 | ^CONDUCT[.]md$ 13 | ^CONTRIBUTING[.]md$ 14 | ^docs 15 | ^pkgdown 16 | 17 | #---------------------------- 18 | # devtools 19 | #---------------------------- 20 | ^revdep 21 | 22 | #---------------------------- 23 | # Travis-CI et al. 24 | #---------------------------- 25 | ^[.]travis[.]yml$ 26 | ^travis-tool[.]sh$ 27 | ^pkg-build[.]sh$ 28 | ^appveyor[.]yml$ 29 | ^covr-utils.R$ 30 | ^[.]covr[.]R$ 31 | ^[.]covr[.]rds$ 32 | 33 | #---------------------------- 34 | # R related 35 | #---------------------------- 36 | Rplots.pdf$ 37 | ^cran-comments[.].*$ 38 | ^vignettes/.*[.](pdf|PDF)$ 39 | ^vignettes/.*[.](r|R)$ 40 | ^vignettes/[.]install_extras$ 41 | ^Makefile$ 42 | ^incl 43 | ^NAMESPACE,.*[.]txt$ 44 | ^nohup.*$ 45 | ^[.]R 46 | ^[.]benchmark 47 | ^[.]devel 48 | ^[.]test 49 | ^[.]check 50 | ^[.]local 51 | ^.*[.]tar[.]gz$ 52 | 53 | #---------------------------- 54 | # Package specific 55 | #---------------------------- 56 | ^[.]BatchJobs[.]R$ 57 | [.]future 58 | 59 | #---------------------------- 60 | # Miscellaneous 61 | #---------------------------- 62 | ^.ghi 63 | ^.issues 64 | 65 | ^.*\.Rproj$ 66 | ^\.Rproj\.user$ 67 | -------------------------------------------------------------------------------- /.Rinstignore: -------------------------------------------------------------------------------- 1 | # Certain LaTeX files (e.g. bib, bst, sty) must be part of the build 2 | # such that they are available for R CMD check. These are excluded 3 | # from the install using .Rinstignore in the top-level directory 4 | # such as this one. 5 | doc/.*[.](bib|bst|sty)$ 6 | -------------------------------------------------------------------------------- /.github/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve (Please use future's 'Discussions' for Q&A) 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | (Please use for Q&A) 10 | 11 | **Describe the bug** 12 | A clear and concise description of what the bug is. 13 | 14 | **Reproduce example** 15 | A reproducible example using R code. 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Session information** 21 | Please share your session information, e.g. 22 | 23 | ```r 24 | > sessionInfo() 25 | ``` 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | lank_issues_enabled: true 2 | contact_links: 3 | - name: Support & Discussions 4 | url: https://github.com/HenrikBengtsson/future/discussions/ 5 | about: Got a question? Something is not working? Want to share an idea? 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project (Please use future's 'Discussions' for Q&A) 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | (Please use for Q&A) 10 | 11 | **Wish or feature request** 12 | A clear and concise description of what the problem is. For example, I would like to be able to ... 13 | -------------------------------------------------------------------------------- /.github/workflows/R-CMD-check.yaml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | 3 | name: R-CMD-check 4 | 5 | jobs: 6 | R-CMD-check: 7 | if: "! contains(github.event.head_commit.message, '[ci skip]')" 8 | 9 | timeout-minutes: 30 10 | 11 | runs-on: ${{ matrix.config.os }} 12 | 13 | name: ${{ matrix.config.os }} (${{ matrix.config.r }}) ${{ matrix.config.label }} 14 | 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | config: 19 | - {os: windows-latest, r: 'devel' } 20 | - {os: windows-latest, r: 'release' } 21 | - {os: windows-latest, r: 'oldrel' } 22 | # - {os: macOS-latest, r: 'devel' } 23 | - {os: macOS-latest, r: 'release' } 24 | - {os: macOS-latest, r: 'oldrel' } 25 | - {os: ubuntu-latest, r: 'devel' } 26 | - {os: ubuntu-latest, r: 'release' } 27 | - {os: ubuntu-latest, r: 'oldrel' } 28 | - {os: ubuntu-latest, r: 'oldrel-1' } 29 | - {os: ubuntu-latest, r: 'oldrel-2' } 30 | - {os: ubuntu-latest, r: '3.6' } 31 | - {os: ubuntu-latest, r: 'release' , language: ko, label: ko } 32 | - {os: ubuntu-latest, r: 'release', future_version: develop, label: 'w/ future-develop' } 33 | - {os: ubuntu-latest, r: 'release', future_version: feature/evalFuture-3, label: 'w/ future-feature/evalFuture-3' } 34 | 35 | env: 36 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 37 | R_KEEP_PKG_SOURCE: yes 38 | R_REMOTES_NO_ERRORS_FROM_WARNINGS: true 39 | ## Test in other locale (optional) 40 | LANGUAGE: ${{ matrix.config.language }} 41 | ## R CMD check 42 | _R_CHECK_CRAN_INCOMING_: false 43 | _R_CHECK_MATRIX_DATA_: true 44 | _R_CHECK_SUGGESTS_ONLY_: true 45 | _R_CHECK_THINGS_IN_TEMP_DIR_: true 46 | ## Specific to 'rcmdcheck' 47 | RCMDCHECK_ERROR_ON: note 48 | ## Specific to futures 49 | R_FUTURE_RNG_ONMISUSE: error 50 | R_FUTURE_VERSION: ${{ matrix.config.future_version }} 51 | 52 | steps: 53 | - uses: actions/checkout@v4 54 | 55 | - uses: r-lib/actions/setup-pandoc@v2 56 | 57 | - uses: r-lib/actions/setup-r@v2 58 | with: 59 | r-version: ${{ matrix.config.r }} 60 | http-user-agent: ${{ matrix.config.http-user-agent }} 61 | use-public-rspm: true 62 | 63 | - uses: r-lib/actions/setup-r-dependencies@v2 64 | with: 65 | extra-packages: any::rcmdcheck 66 | needs: check 67 | 68 | - name: Install package itself (special case) 69 | run: | 70 | install.packages(".", repos = NULL, type = "source") ## needed by parallel workers 71 | shell: Rscript {0} 72 | 73 | - name: Test with specific future version? 74 | run: | 75 | future_version <- Sys.getenv("R_FUTURE_VERSION") 76 | if (nzchar(future_version)) { 77 | install.packages("remotes") 78 | remotes::install_github("futureverse/future", ref=future_version) 79 | } 80 | shell: Rscript {0} 81 | 82 | - name: Session info 83 | run: | 84 | options(width = 100) 85 | parallelly::availableCores(which = "all") 86 | sapply(c(physical_cores = FALSE, logical_cores = TRUE), parallel::detectCores) 87 | if (require(RhpcBLASctl, quietly=TRUE)) c(get_num_procs = get_num_procs(), get_num_cores = get_num_cores(), blas_get_num_procs = blas_get_num_procs(), omp_get_num_procs = omp_get_num_procs(), omp_get_max_threads = omp_get_max_threads()) 88 | capabilities() 89 | pkgs <- installed.packages()[, "Package"] 90 | sessioninfo::session_info(pkgs, include_base = TRUE) 91 | ## Verify LANGUAGE settings by generating a translatable error 92 | cat(sprintf("LANGUAGE=%s\n", sQuote(Sys.getenv("LANGUAGE")))) 93 | cat(sprintf("locales: %s\n", sQuote(Sys.getlocale()))) 94 | tryCatch(log("a"), error = conditionMessage) 95 | shell: Rscript {0} 96 | 97 | - name: Check 98 | run: | 99 | if (nzchar(Sys.getenv("R_FUTURE_PLAN"))) Sys.setenv(RCMDCHECK_ERROR_ON = "error") 100 | rcmdcheck::rcmdcheck( 101 | args = c("--no-manual", "--as-cran"), 102 | check_dir = "check" 103 | ) 104 | shell: Rscript {0} 105 | 106 | - name: Upload check results 107 | if: failure() 108 | uses: actions/upload-artifact@v4 109 | with: 110 | name: ${{ runner.os }}-r${{ matrix.config.r }}-results 111 | path: check 112 | -------------------------------------------------------------------------------- /.github/workflows/future_tests.yaml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | 3 | name: future_tests 4 | 5 | jobs: 6 | future_tests: 7 | if: "! contains(github.event.head_commit.message, '[ci skip]')" 8 | 9 | timeout-minutes: 15 10 | 11 | runs-on: ubuntu-latest 12 | 13 | name: future.plan=${{ matrix.future.plan }} (${{ matrix.future.label }}) 14 | 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | future: 19 | - { plan: 'future.batchtools::batchtools_local' } 20 | 21 | env: 22 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 23 | R_REMOTES_NO_ERRORS_FROM_WARNINGS: true 24 | ## R CMD check 25 | _R_CHECK_LENGTH_1_CONDITION_: true 26 | _R_CHECK_LENGTH_1_LOGIC2_: true 27 | _R_CHECK_MATRIX_DATA_: true 28 | _R_CHECK_CRAN_INCOMING_: false 29 | ## Specific to futures 30 | R_FUTURE_RNG_ONMISUSE: error 31 | 32 | steps: 33 | - uses: actions/checkout@v4 34 | 35 | - uses: r-lib/actions/setup-r@v2 36 | with: 37 | use-public-rspm: true 38 | 39 | - uses: r-lib/actions/setup-r-dependencies@v2 40 | with: 41 | extra-packages: | 42 | any::rcmdcheck 43 | any::remotes 44 | needs: check 45 | 46 | - name: Install R package dependencies 47 | run: | 48 | remotes::install_deps(dependencies = TRUE) 49 | install.packages(".", repos=NULL, type="source") ## needed by parallel workers 50 | shell: Rscript {0} 51 | 52 | - name: Install 'future.tests' and any backend R packages 53 | run: | 54 | remotes::install_cran("future.tests") 55 | remotes::install_github("HenrikBengtsson/future.tests", ref="develop") 56 | shell: Rscript {0} 57 | 58 | - name: Session info 59 | run: | 60 | options(width = 100) 61 | pkgs <- installed.packages()[, "Package"] 62 | sessioninfo::session_info(pkgs, include_base = TRUE) 63 | shell: Rscript {0} 64 | 65 | - name: Check future backend '${{ matrix.future.plan }}' 66 | run: | 67 | R CMD build --no-build-vignettes --no-manual . 68 | R CMD INSTALL *.tar.gz 69 | Rscript -e future.tests::check --args --test-plan=${{ matrix.future.plan }} 70 | 71 | - name: Upload check results 72 | if: failure() 73 | uses: actions/upload-artifact@v4 74 | with: 75 | name: ${{ runner.os }}-r${{ matrix.future.plan }}-results 76 | path: check 77 | -------------------------------------------------------------------------------- /.github/workflows/test-coverage.yaml: -------------------------------------------------------------------------------- 1 | on: [push] 2 | 3 | name: covr 4 | 5 | jobs: 6 | covr: 7 | if: "! contains(github.event.head_commit.message, '[ci skip]')" 8 | 9 | timeout-minutes: 45 10 | 11 | runs-on: ubuntu-latest 12 | 13 | name: covr 14 | 15 | strategy: 16 | fail-fast: false 17 | 18 | env: 19 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 20 | R_KEEP_PKG_SOURCE: yes 21 | R_REMOTES_NO_ERRORS_FROM_WARNINGS: true 22 | ## Test in other locale (optional) 23 | LANGUAGE: ${{ matrix.config.language }} 24 | ## R CMD check 25 | _R_CHECK_CRAN_INCOMING_: false 26 | _R_CHECK_MATRIX_DATA_: true 27 | _R_CHECK_SUGGESTS_ONLY_: true 28 | _R_CHECK_THINGS_IN_TEMP_DIR_: true 29 | RCMDCHECK_ERROR_ON: note 30 | ## Specific to futures 31 | R_FUTURE_RNG_ONMISUSE: error 32 | 33 | steps: 34 | - uses: actions/checkout@v4 35 | 36 | - uses: r-lib/actions/setup-r@v2 37 | with: 38 | use-public-rspm: true 39 | 40 | - uses: r-lib/actions/setup-r-dependencies@v2 41 | with: 42 | extra-packages: | 43 | any::rcmdcheck 44 | any::remotes 45 | any::sessioninfo 46 | any::covr 47 | needs: check 48 | 49 | - name: Install dependencies 50 | run: | 51 | remotes::install_deps(dependencies = TRUE) 52 | install.packages(".", repos=NULL, type="source") 53 | shell: Rscript {0} 54 | 55 | - name: Session info 56 | run: | 57 | options(width = 100) 58 | pkgs <- installed.packages()[, "Package"] 59 | sessioninfo::session_info(pkgs, include_base = TRUE) 60 | shell: Rscript {0} 61 | 62 | - name: Test coverage 63 | run: | 64 | ## 1. Get 'Repository Upload Token' from Codecov: 65 | ## https://app.codecov.io/gh///settings 66 | ## 2. Set 'CODECOV_TOKEN' in GitHub Secrets: 67 | ## https://github.com///settings/environments/ 68 | coverage <- covr::package_coverage(quiet = FALSE) 69 | print(coverage) 70 | covr::codecov(coverage = coverage, token="${{secrets.CODECOV_TOKEN}}") 71 | shell: Rscript {0} 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .Rhistory 2 | *~ 3 | **/*~ 4 | .R 5 | .benchmark 6 | .check 7 | .devel 8 | .test 9 | *.o 10 | *.dll 11 | *.Rout 12 | .RData 13 | *.Rproj* 14 | *.swp 15 | .covr.rds 16 | .future 17 | .ghi 18 | .issues 19 | .make 20 | .local 21 | revdep/data.sqlite 22 | revdep/checks/* 23 | revdep/library/* 24 | docs/ 25 | .Rdump 26 | -------------------------------------------------------------------------------- /CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | # Contributing to the 'future.batchtools' package 3 | 4 | This Git repository uses the [Git Flow](https://nvie.com/posts/a-successful-git-branching-model/) branching model (the [`git flow`](https://github.com/petervanderdoes/gitflow-avh) extension is useful for this). The [`develop`](https://github.com/futureverse/future.batchtools/tree/develop) branch contains the latest contributions and other code that will appear in the next release, and the [`master`](https://github.com/futureverse/future.batchtools) branch contains the code of the latest release, which is exactly what is currently on [CRAN](https://cran.r-project.org/package=future.batchtools). 5 | 6 | Contributing to this package is easy. Just send a [pull request](https://help.github.com/articles/using-pull-requests/). When you send your PR, make sure `develop` is the destination branch on the [future.batchtools repository](https://github.com/futureverse/future.batchtools). Your PR should pass `R CMD check --as-cran`, which will also be checked by GitHub Actions and when the PR is submitted. 7 | 8 | We abide to the [Code of Conduct](https://www.contributor-covenant.org/version/2/0/code_of_conduct/) of Contributor Covenant. 9 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: future.batchtools 2 | Version: 0.12.2-9000 3 | Depends: 4 | R (>= 3.2.0), 5 | parallelly, 6 | future (>= 1.49.0) 7 | Imports: 8 | batchtools (>= 0.9.16), 9 | utils 10 | Suggests: 11 | globals, 12 | future.apply, 13 | listenv, 14 | markdown, 15 | R.rsp 16 | VignetteBuilder: 17 | R.rsp 18 | Title: A Future API for Parallel and Distributed Processing using 'batchtools' 19 | Authors@R: c(person("Henrik", "Bengtsson", 20 | role = c("aut", "cre", "cph"), 21 | email = "henrikb@braju.com", 22 | comment = c(ORCID = "0000-0002-7579-5165"))) 23 | Description: Implementation of the Future API on top of the 'batchtools' package. 24 | This allows you to process futures, as defined by the 'future' package, 25 | in parallel out of the box, not only on your local machine or ad-hoc 26 | cluster of machines, but also via high-performance compute ('HPC') job 27 | schedulers such as 'LSF', 'OpenLava', 'Slurm', 'SGE', and 'TORQUE' / 'PBS', 28 | e.g. 'y <- future.apply::future_lapply(files, FUN = process)'. 29 | License: LGPL (>= 2.1) 30 | LazyLoad: TRUE 31 | URL: https://future.batchtools.futureverse.org, https://github.com/futureverse/future.batchtools 32 | BugReports: https://github.com/futureverse/future.batchtools/issues 33 | RoxygenNote: 7.3.2 34 | Roxygen: list(markdown = TRUE) 35 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include .make/Makefile 2 | 3 | spelling: 4 | $(R_SCRIPT) -e "spelling::spell_check_files(c('NEWS', dir('vignettes', pattern='[.]rsp', full.names=TRUE)), ignore=readLines('inst/WORDLIST', warn=FALSE))" 5 | 6 | future.tests/%: 7 | $(R_SCRIPT) -e "future.tests::check" --args --test-plan=$* 8 | 9 | future.tests: future.tests/future.batchtools\:\:batchtools_local 10 | 11 | spelling: 12 | $(R_SCRIPT) -e "spelling::spell_check_package()" 13 | $(R_SCRIPT) -e "spelling::spell_check_files(c('NEWS', dir('vignettes', pattern='[.]rsp', full.names=TRUE)), ignore=readLines('inst/WORDLIST', warn=FALSE))" 14 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | S3method(add_finalizer,BatchtoolsFuture) 4 | S3method(cancel,BatchtoolsFuture) 5 | S3method(delete,BatchtoolsFuture) 6 | S3method(loggedError,BatchtoolsFuture) 7 | S3method(loggedOutput,BatchtoolsFuture) 8 | S3method(nbrOfFreeWorkers,batchtools) 9 | S3method(nbrOfFreeWorkers,batchtools_multiprocess) 10 | S3method(nbrOfFreeWorkers,batchtools_uniprocess) 11 | S3method(nbrOfWorkers,batchtools) 12 | S3method(nbrOfWorkers,batchtools_multicore) 13 | S3method(nbrOfWorkers,batchtools_uniprocess) 14 | S3method(print,BatchtoolsFuture) 15 | S3method(registerFuture,BatchtoolsFuture) 16 | S3method(registerFuture,BatchtoolsUniprocessFuture) 17 | S3method(registerFuture,default) 18 | S3method(resolved,BatchtoolsFuture) 19 | S3method(result,BatchtoolsFuture) 20 | S3method(run,BatchtoolsFuture) 21 | S3method(unregisterFuture,BatchtoolsFuture) 22 | S3method(unregisterFuture,BatchtoolsUniprocessFuture) 23 | S3method(unregisterFuture,default) 24 | S3method(waitForWorker,BatchtoolsFuture) 25 | S3method(waitForWorker,BatchtoolsUniprocessFuture) 26 | S3method(waitForWorker,default) 27 | export("%resources%") 28 | export(BatchtoolsBashFuture) 29 | export(BatchtoolsCustomFuture) 30 | export(BatchtoolsFuture) 31 | export(BatchtoolsFutureError) 32 | export(BatchtoolsInteractiveFuture) 33 | export(BatchtoolsLocalFuture) 34 | export(BatchtoolsLsfFuture) 35 | export(BatchtoolsMulticoreFuture) 36 | export(BatchtoolsMultiprocessFuture) 37 | export(BatchtoolsOpenLavaFuture) 38 | export(BatchtoolsSGEFuture) 39 | export(BatchtoolsSSHFuture) 40 | export(BatchtoolsSlurmFuture) 41 | export(BatchtoolsTemplateFuture) 42 | export(BatchtoolsTorqueFuture) 43 | export(BatchtoolsUniprocessFuture) 44 | export(batchtools_bash) 45 | export(batchtools_custom) 46 | export(batchtools_interactive) 47 | export(batchtools_local) 48 | export(batchtools_lsf) 49 | export(batchtools_multicore) 50 | export(batchtools_openlava) 51 | export(batchtools_sge) 52 | export(batchtools_slurm) 53 | export(batchtools_ssh) 54 | export(batchtools_torque) 55 | export(loggedError) 56 | export(loggedOutput) 57 | importFrom(batchtools,Worker) 58 | importFrom(batchtools,batchExport) 59 | importFrom(batchtools,batchMap) 60 | importFrom(batchtools,cfBrewTemplate) 61 | importFrom(batchtools,cfReadBrewTemplate) 62 | importFrom(batchtools,clearRegistry) 63 | importFrom(batchtools,findConfFile) 64 | importFrom(batchtools,findTemplateFile) 65 | importFrom(batchtools,getErrorMessages) 66 | importFrom(batchtools,getLog) 67 | importFrom(batchtools,getStatus) 68 | importFrom(batchtools,loadResult) 69 | importFrom(batchtools,makeClusterFunctions) 70 | importFrom(batchtools,makeClusterFunctionsInteractive) 71 | importFrom(batchtools,makeClusterFunctionsLSF) 72 | importFrom(batchtools,makeClusterFunctionsMulticore) 73 | importFrom(batchtools,makeClusterFunctionsOpenLava) 74 | importFrom(batchtools,makeClusterFunctionsSGE) 75 | importFrom(batchtools,makeClusterFunctionsSSH) 76 | importFrom(batchtools,makeClusterFunctionsSlurm) 77 | importFrom(batchtools,makeClusterFunctionsTORQUE) 78 | importFrom(batchtools,makeRegistry) 79 | importFrom(batchtools,makeSubmitJobResult) 80 | importFrom(batchtools,removeRegistry) 81 | importFrom(batchtools,saveRegistry) 82 | importFrom(batchtools,setJobNames) 83 | importFrom(batchtools,submitJobs) 84 | importFrom(batchtools,waitForJobs) 85 | importFrom(future,Future) 86 | importFrom(future,FutureError) 87 | importFrom(future,cancel) 88 | importFrom(future,getExpression) 89 | importFrom(future,getGlobalsAndPackages) 90 | importFrom(future,nbrOfFreeWorkers) 91 | importFrom(future,nbrOfWorkers) 92 | importFrom(future,plan) 93 | importFrom(future,resolved) 94 | importFrom(future,result) 95 | importFrom(future,run) 96 | importFrom(future,tweak) 97 | importFrom(parallelly,availableCores) 98 | importFrom(parallelly,availableWorkers) 99 | importFrom(utils,capture.output) 100 | importFrom(utils,file_test) 101 | importFrom(utils,sessionInfo) 102 | importFrom(utils,str) 103 | importFrom(utils,tail) 104 | -------------------------------------------------------------------------------- /OVERVIEW.md: -------------------------------------------------------------------------------- 1 | <% 2 | ## Reuse the future vignette 3 | md <- R.rsp::rstring(file="vignettes/future.batchtools.md.rsp", postprocess=FALSE) 4 | 5 | ## Drop the header, i.e. anything before the first "H2" header 6 | md <- unlist(strsplit(md, split="\n", fixed=TRUE)) 7 | row <- grep("^## ", md)[1] 8 | if (!is.na(row)) md <- md[-seq_len(row-1)] 9 | 10 | ## Drop the footer, i.e. anything after the first horizontal line 11 | row <- grep("^---", md)[1] 12 | if (!is.na(row)) md <- md[seq_len(row-1)] 13 | 14 | ## Output 15 | cat(md, sep="\n") 16 | %> 17 | -------------------------------------------------------------------------------- /R/000.import.R: -------------------------------------------------------------------------------- 1 | import_from <- function(name, mode = "function", default = NULL, package) { 2 | ns <- getNamespace(package) 3 | if (exists(name, mode = mode, envir = ns, inherits = FALSE)) { 4 | get(name, mode = mode, envir = ns, inherits = FALSE) 5 | } else if (!is.null(default)) { 6 | default 7 | } else { 8 | stop(sprintf("No such '%s' %s: %s()", package, mode, name)) 9 | } 10 | } 11 | 12 | import_future <- function(...) { 13 | import_from(..., package = "future") 14 | } 15 | 16 | import_parallelly <- function(...) { 17 | import_from(..., package = "parallelly") 18 | } 19 | -------------------------------------------------------------------------------- /R/001.import_future_functions.R: -------------------------------------------------------------------------------- 1 | ## To be imported from 'future', if available 2 | readImmediateConditions <- NULL 3 | signalEarly <- NULL 4 | FutureRegistry <- NULL 5 | sQuoteLabel <- NULL 6 | .debug <- NULL 7 | 8 | ## Import private functions from 'future' 9 | import_future_functions <- function() { 10 | readImmediateConditions <<- import_future("readImmediateConditions") 11 | signalEarly <<- import_future("signalEarly") 12 | FutureRegistry <<- import_future("FutureRegistry") 13 | 14 | ## future (>= 1.49.0) 15 | sQuoteLabel <<- import_future("sQuoteLabel") 16 | 17 | .debug <<- import_future(".debug", mode = "environment", default = new.env(parent = emptyenv())) 18 | } 19 | -------------------------------------------------------------------------------- /R/BatchtoolsCustomFuture-class.R: -------------------------------------------------------------------------------- 1 | #' @rdname BatchtoolsFuture 2 | #' @export 3 | BatchtoolsCustomFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 4 | if (substitute) expr <- substitute(expr) 5 | assert_no_positional_args_but_first() 6 | 7 | future <- BatchtoolsFuture(expr = expr, substitute = FALSE, envir = envir, ...) 8 | future <- structure(future, class = c("BatchtoolsCustomFuture", class(future))) 9 | 10 | future 11 | } 12 | 13 | 14 | 15 | #' @rdname BatchtoolsFuture 16 | #' @export 17 | BatchtoolsBashFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 18 | if (substitute) expr <- substitute(expr) 19 | assert_no_positional_args_but_first() 20 | 21 | future <- BatchtoolsCustomFuture(expr = expr, substitute = FALSE, envir = envir, ..., workers = 1L) 22 | future <- structure(future, class = c("BatchtoolsBashFuture", "BatchtoolsUniprocessFuture", class(future))) 23 | 24 | future 25 | } 26 | -------------------------------------------------------------------------------- /R/BatchtoolsFutureError.R: -------------------------------------------------------------------------------- 1 | #' FutureError class for errors related to BatchtoolsFuture:s 2 | #' 3 | #' @param \ldots Arguments passed to [FutureError][future::FutureError]. 4 | #' 5 | #' @export 6 | #' @importFrom future FutureError 7 | #' 8 | #' @keywords internal 9 | BatchtoolsFutureError <- function(...) { 10 | error <- FutureError(...) 11 | class(error) <- c("BatchtoolsFutureError", class(error)) 12 | error 13 | } 14 | -------------------------------------------------------------------------------- /R/BatchtoolsMultiprocessFuture-class.R: -------------------------------------------------------------------------------- 1 | #' @rdname BatchtoolsFuture 2 | #' @export 3 | BatchtoolsMultiprocessFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 4 | if (substitute) expr <- substitute(expr) 5 | assert_no_positional_args_but_first() 6 | 7 | future <- BatchtoolsFuture(expr = expr, substitute = FALSE, envir = envir, ...) 8 | future <- structure(future, class = c("BatchtoolsMultiprocessFuture", class(future))) 9 | 10 | future 11 | } 12 | 13 | 14 | #' @rdname BatchtoolsFuture 15 | #' @export 16 | BatchtoolsMulticoreFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 17 | if (substitute) expr <- substitute(expr) 18 | assert_no_positional_args_but_first() 19 | 20 | future <- BatchtoolsMultiprocessFuture(expr = expr, substitute = FALSE, envir = envir, ...) 21 | future <- structure(future, class = c("BatchtoolsMulticoreFuture", class(future))) 22 | 23 | future 24 | } 25 | 26 | 27 | #' @rdname BatchtoolsFuture 28 | #' @export 29 | BatchtoolsTemplateFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 30 | if (substitute) expr <- substitute(expr) 31 | assert_no_positional_args_but_first() 32 | 33 | future <- BatchtoolsMultiprocessFuture(expr = expr, substitute = FALSE, envir = envir, ...) 34 | future <- structure(future, class = c("BatchtoolsTemplateFuture", class(future))) 35 | 36 | future 37 | } 38 | 39 | 40 | #' @rdname BatchtoolsFuture 41 | #' @export 42 | BatchtoolsLsfFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 43 | if (substitute) expr <- substitute(expr) 44 | 45 | future <- BatchtoolsTemplateFuture(expr = expr, substitute = FALSE, envir = envir, ...) 46 | future <- structure(future, class = c("BatchtoolsLsfFuture", class(future))) 47 | 48 | future 49 | } 50 | 51 | 52 | #' @rdname BatchtoolsFuture 53 | #' @export 54 | BatchtoolsOpenLavaFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 55 | if (substitute) expr <- substitute(expr) 56 | assert_no_positional_args_but_first() 57 | 58 | future <- BatchtoolsTemplateFuture(expr = expr, substitute = FALSE, envir = envir, ...) 59 | future <- structure(future, class = c("BatchtoolsOpenLavaFuture", class(future))) 60 | 61 | future 62 | } 63 | 64 | 65 | #' @rdname BatchtoolsFuture 66 | #' @export 67 | BatchtoolsSGEFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 68 | if (substitute) expr <- substitute(expr) 69 | assert_no_positional_args_but_first() 70 | 71 | future <- BatchtoolsTemplateFuture(expr = expr, substitute = FALSE, envir = envir, ...) 72 | future <- structure(future, class = c("BatchtoolsSGEFuture", class(future))) 73 | 74 | future 75 | } 76 | 77 | 78 | #' @rdname BatchtoolsFuture 79 | #' @export 80 | BatchtoolsSlurmFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 81 | if (substitute) expr <- substitute(expr) 82 | 83 | future <- BatchtoolsTemplateFuture(expr = expr, substitute = FALSE, envir = envir, ...) 84 | future <- structure(future, class = c("BatchtoolsSlurmFuture", class(future))) 85 | 86 | future 87 | } 88 | 89 | 90 | #' @rdname BatchtoolsFuture 91 | #' @export 92 | BatchtoolsTorqueFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 93 | if (substitute) expr <- substitute(expr) 94 | assert_no_positional_args_but_first() 95 | 96 | future <- BatchtoolsTemplateFuture(expr = expr, substitute = FALSE, envir = envir, ...) 97 | future <- structure(future, class = c("BatchtoolsTorqueFuture", class(future))) 98 | 99 | future 100 | } 101 | -------------------------------------------------------------------------------- /R/BatchtoolsSSHFuture-class.R: -------------------------------------------------------------------------------- 1 | #' @rdname BatchtoolsFuture 2 | #' @importFrom batchtools makeClusterFunctionsSSH 3 | #' @importFrom parallelly availableCores 4 | #' @export 5 | BatchtoolsSSHFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), workers = availableCores(), ...) { 6 | if (substitute) expr <- substitute(expr) 7 | assert_no_positional_args_but_first() 8 | 9 | if (is.null(workers)) workers <- availableCores() 10 | stopifnot(is.numeric(workers), length(workers) == 1L, !is.na(workers), is.finite(workers), workers >= 1L) 11 | 12 | dotdotdot <- list(...) 13 | 14 | ## WORKAROUND: 'max.load' cannot be +Inf, because that'll lead to: 15 | ## 16 | ## Error in sample.int(x, size, replace, prob) : 17 | ## too few positive probabilities 18 | ## 19 | ## in the submitJob() function created by makeClusterFunctionsSSH(). 20 | ## /HB 2022-12-12 21 | ssh_worker <- list(Worker$new( 22 | "localhost", 23 | ncpus = 1L, 24 | max.load = .Machine$double.xmax ## +Inf fails 25 | )) 26 | 27 | keep <- which(names(dotdotdot) %in% names(formals(makeClusterFunctionsSSH))) 28 | args <- c(list(workers = ssh_worker), dotdotdot[keep]) 29 | cluster.functions <- do.call(makeClusterFunctionsSSH, args = args) 30 | 31 | ## Drop used '...' arguments 32 | if (length(keep) > 0) dotdotdot <- dotdotdot[-keep] 33 | 34 | args <- list( 35 | expr = quote(expr), ## Avoid 'expr' being resolved by do.call() 36 | substitute = FALSE, envir = envir, 37 | workers = workers, 38 | cluster.functions = cluster.functions 39 | ) 40 | if (length(dotdotdot) > 0) args <- c(args, dotdotdot) 41 | 42 | future <- do.call(BatchtoolsCustomFuture, args = args) 43 | 44 | future <- structure(future, class = c("BatchtoolsSSHFuture", class(future))) 45 | 46 | future$workers <- workers 47 | 48 | future 49 | } 50 | -------------------------------------------------------------------------------- /R/BatchtoolsSSHRegistry.R: -------------------------------------------------------------------------------- 1 | BatchtoolsSSHRegistry <- local({ 2 | last <- NULL 3 | cluster <- NULL 4 | 5 | function(action = c("get", "start", "stop"), workers = NULL, makeCluster = .makeCluster, ...) { 6 | action <- match.arg(action, choices = c("get", "start", "stop")) 7 | 8 | if (is.null(workers)) { 9 | } else if (is.numeric(workers)) { 10 | workers <- as.integer(workers) 11 | stop_if_not(length(workers) == 1, !is.na(workers), is.finite(workers)) 12 | } else if (is.character(workers)) { 13 | stop_if_not(length(workers) >= 1, !anyNA(workers)) 14 | workers <- sort(workers) 15 | } else { 16 | stopf("Unknown mode of argument 'workers': %s", mode(workers)) 17 | } 18 | 19 | if (length(cluster) == 0L && action != "stop") { 20 | cluster <<- makeCluster(workers, ...) 21 | last <<- workers 22 | } 23 | 24 | if (action == "get") { 25 | return(cluster) 26 | } else if (action == "start") { 27 | ## Already setup? 28 | if (!identical(workers, last)) { 29 | BatchtoolsSSHRegistry(action = "stop") 30 | cluster <<- makeCluster(workers, ...) 31 | last <<- workers 32 | } 33 | } else if (action == "stop") { 34 | cluster <<- NULL 35 | last <<- NULL 36 | } 37 | 38 | invisible(cluster) 39 | } 40 | }) ## BatchtoolsSSHRegistry() 41 | 42 | 43 | #' @importFrom batchtools Worker 44 | .makeCluster <- function(workers, ...) { 45 | if (is.numeric(workers)) { 46 | stop_if_not(length(workers) == 1L, !is.na(workers), is.finite(workers), workers >= 1) 47 | workers <- rep("localhost", times = workers) 48 | } 49 | if (length(workers) == 0L) return(NULL) 50 | ncpus <- table(workers) 51 | 52 | mapply(names(ncpus), ncpus, FUN = function(hostname, ncpus) { 53 | Worker$new(hostname, ncpus = ncpus, max.load = Inf) 54 | }) 55 | } 56 | -------------------------------------------------------------------------------- /R/BatchtoolsUniprocessFuture-class.R: -------------------------------------------------------------------------------- 1 | #' @rdname BatchtoolsFuture 2 | #' @export 3 | BatchtoolsUniprocessFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 4 | if (substitute) expr <- substitute(expr) 5 | assert_no_positional_args_but_first() 6 | 7 | future <- BatchtoolsFuture(expr = expr, substitute = FALSE, envir = envir, ..., workers = 1L) 8 | future <- structure(future, class = c("BatchtoolsUniprocessFuture", class(future))) 9 | 10 | future 11 | } 12 | 13 | 14 | #' @rdname BatchtoolsFuture 15 | #' @export 16 | BatchtoolsLocalFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 17 | if (substitute) expr <- substitute(expr) 18 | assert_no_positional_args_but_first() 19 | 20 | future <- BatchtoolsUniprocessFuture(expr = expr, substitute = FALSE, envir = envir, ...) 21 | future <- structure(future, class = c("BatchtoolsLocalFuture", class(future))) 22 | 23 | future 24 | } 25 | 26 | 27 | #' @rdname BatchtoolsFuture 28 | #' @export 29 | BatchtoolsInteractiveFuture <- function(expr = NULL, substitute = TRUE, envir = parent.frame(), ...) { 30 | if (substitute) expr <- substitute(expr) 31 | assert_no_positional_args_but_first() 32 | 33 | future <- BatchtoolsUniprocessFuture(expr = expr, substitute = FALSE, envir = envir, ...) 34 | future <- structure(future, class = c("BatchtoolsInteractiveFuture", class(future))) 35 | 36 | future 37 | } 38 | -------------------------------------------------------------------------------- /R/batchtools_bash.R: -------------------------------------------------------------------------------- 1 | #' @inheritParams batchtools_custom 2 | #' @inheritParams batchtools_template 3 | #' 4 | #' @export 5 | batchtools_bash <- function(..., envir = parent.frame(), template = "bash") { 6 | cf <- makeClusterFunctionsBash(template = template) 7 | future <- BatchtoolsBashFuture(..., envir = envir, cluster.functions = cf) 8 | if (!future$lazy) future <- run(future) 9 | invisible(future) 10 | } 11 | class(batchtools_bash) <- c( 12 | "batchtools_bash", "batchtools_custom", 13 | "batchtools_uniprocess", "batchtools", 14 | "uniprocess", "future", "function" 15 | ) 16 | attr(batchtools_bash, "tweakable") <- c("finalize") 17 | attr(batchtools_bash, "untweakable") <- c("workers") 18 | 19 | 20 | #' @importFrom batchtools cfReadBrewTemplate cfBrewTemplate makeClusterFunctions makeSubmitJobResult 21 | #' @importFrom utils file_test 22 | makeClusterFunctionsBash <- function(template = "bash") { 23 | bin <- Sys.which("bash") 24 | stop_if_not(file_test("-f", bin), file_test("-x", bin)) 25 | 26 | template <- find_template_file(template) 27 | template_text <- cfReadBrewTemplate(template) 28 | 29 | submitJob <- function(reg, jc) { 30 | stop_if_not(inherits(reg, "Registry")) 31 | stop_if_not(inherits(jc, "JobCollection")) 32 | 33 | script <- cfBrewTemplate(reg, text = template_text, jc = jc) 34 | output <- system2(bin, args = c(script), stdout = TRUE, stderr = TRUE) 35 | debug <- isTRUE(getOption("future.debug")) 36 | if (debug) { 37 | mdebug_push("makeClusterFunctionsBash() ...") 38 | mdebug(paste(c(output, ""), collapse = "\n")) 39 | on.exit(mdebug_pop()) 40 | } 41 | 42 | status <- attr(output, "status") 43 | if (is.null(status)) { 44 | status <- 0L 45 | batch.id <- sprintf("bash#%d", Sys.getpid()) 46 | } else { 47 | batch.id <- NA_character_ 48 | } 49 | 50 | makeSubmitJobResult(status = status, batch.id = batch.id) 51 | } 52 | 53 | cf <- makeClusterFunctions( 54 | name = "Bash", 55 | submitJob = submitJob, 56 | store.job.collection = TRUE 57 | ) 58 | attr(cf, "template") <- template 59 | attr(cf, "template_text") <- template_text 60 | cf 61 | } 62 | -------------------------------------------------------------------------------- /R/batchtools_custom.R: -------------------------------------------------------------------------------- 1 | #' Batchtools futures for custom batchtools configuration 2 | #' 3 | #' @inheritParams BatchtoolsFuture 4 | #' 5 | #' @param conf.file (character) A batchtools configuration file as for 6 | #' instance returned by [batchtools::findConfFile()]. 7 | #' 8 | #' @param cluster.functions A 9 | #' [ClusterFunctions][batchtools::ClusterFunctions] object. 10 | #' 11 | #' @param \ldots Additional arguments passed to [BatchtoolsFuture()]. 12 | #' 13 | #' @return An object of class `BatchtoolsFuture`. 14 | #' 15 | #' @example incl/batchtools_custom.R 16 | #' 17 | #' @export 18 | #' @importFrom batchtools findConfFile 19 | #' @importFrom utils file_test 20 | batchtools_custom <- function(expr, envir = parent.frame(), substitute = TRUE, 21 | globals = TRUE, 22 | label = NULL, 23 | resources = list(), 24 | workers = NULL, 25 | conf.file = findConfFile(), 26 | cluster.functions = NULL, 27 | registry = list(), 28 | ...) { 29 | if (substitute) expr <- substitute(expr) 30 | 31 | future <- BatchtoolsCustomFuture( 32 | expr = expr, envir = envir, substitute = FALSE, 33 | globals = globals, 34 | label = label, 35 | resources = resources, 36 | conf.file = conf.file, 37 | cluster.functions = cluster.functions, 38 | workers = workers, 39 | registry = registry, 40 | ... 41 | ) 42 | 43 | if (!future$lazy) future <- run(future) 44 | 45 | invisible(future) 46 | } 47 | class(batchtools_custom) <- c( 48 | "batchtools_custom", "batchtools_multiprocess", "batchtools", 49 | "multiprocess", "future", "function" 50 | ) 51 | attr(batchtools_custom, "tweakable") <- c("finalize") 52 | -------------------------------------------------------------------------------- /R/batchtools_interactive.R: -------------------------------------------------------------------------------- 1 | #' @inheritParams BatchtoolsUniprocessFuture 2 | #' 3 | #' @importFrom batchtools makeClusterFunctionsInteractive 4 | #' @export 5 | batchtools_interactive <- function(..., envir = parent.frame()) { 6 | cf <- makeClusterFunctionsInteractive(external = FALSE) 7 | future <- BatchtoolsInteractiveFuture(..., envir = envir, cluster.functions = cf) 8 | if (!future$lazy) future <- run(future) 9 | invisible(future) 10 | } 11 | class(batchtools_interactive) <- c( 12 | "batchtools_interactive", "batchtools_uniprocess", "batchtools", 13 | "uniprocess", "future", "function" 14 | ) 15 | attr(batchtools_interactive, "tweakable") <- c("finalize") 16 | attr(batchtools_interactive, "untweakable") <- c("workers") 17 | -------------------------------------------------------------------------------- /R/batchtools_local.R: -------------------------------------------------------------------------------- 1 | #' batchtools local and interactive futures 2 | #' 3 | #' A batchtools local future is an synchronous uniprocess future that 4 | #' will be evaluated in a background R session. 5 | #' A batchtools interactive future is an synchronous uniprocess future 6 | #' that will be evaluated in the current R session (and variables will 7 | #' be assigned to the calling environment rather than to a local one). 8 | #' Both types of futures will block until the futures are resolved. 9 | #' 10 | #' @inheritParams BatchtoolsUniprocessFuture 11 | #' 12 | #' @param \ldots Additional arguments passed to [BatchtoolsUniprocessFuture()]. 13 | #' 14 | #' @return An object of class `BatchtoolsUniprocessFuture`. 15 | #' 16 | #' @details 17 | #' batchtools local futures rely on the batchtools backend set up by 18 | #' \code{\link[batchtools:makeClusterFunctionsInteractive]{batchtools::makeClusterFunctionsInteractive(external = TRUE)}} 19 | #' and batchtools interactive futures on the one set up by 20 | #' [batchtools::makeClusterFunctionsInteractive()]. 21 | #' These are supported by all operating systems. 22 | #' 23 | #' An alternative to batchtools local futures is to use 24 | #' [cluster][future::cluster] futures of the \pkg{future} 25 | #' package with a single local background session, i.e. 26 | #' `plan(cluster, workers = "localhost")`. 27 | #' 28 | #' An alternative to batchtools interactive futures is to use 29 | #' `plan(sequential, split = TRUE)` futures of the \pkg{future} package. 30 | #' 31 | #' @example incl/batchtools_local.R 32 | #' 33 | #' @importFrom batchtools makeClusterFunctionsInteractive 34 | #' @aliases batchtools_interactive batchtools_bash 35 | #' @export 36 | batchtools_local <- function(..., envir = parent.frame()) { 37 | cf <- makeClusterFunctionsInteractive(external = TRUE) 38 | future <- BatchtoolsLocalFuture(..., envir = envir, cluster.functions = cf) 39 | if (!future$lazy) future <- run(future) 40 | invisible(future) 41 | } 42 | class(batchtools_local) <- c( 43 | "batchtools_local", "batchtools_uniprocess", "batchtools", 44 | "uniprocess", "future", "function" 45 | ) 46 | attr(batchtools_local, "tweakable") <- c("finalize") 47 | attr(batchtools_local, "untweakable") <- c("workers") 48 | -------------------------------------------------------------------------------- /R/batchtools_multicore.R: -------------------------------------------------------------------------------- 1 | #' batchtools multicore futures 2 | #' 3 | #' A batchtools multicore future is an asynchronous multiprocess 4 | #' future that will be evaluated in a background R session.\cr 5 | #' \cr 6 | #' _We highly recommend using [future::multisession] 7 | #' (sic!) futures of the \pkg{future} package instead of 8 | #' multicore batchtools futures._ 9 | #' 10 | #' @inheritParams BatchtoolsFuture 11 | #' 12 | #' @param workers The number of multicore processes to be 13 | #' available for concurrent batchtools multicore futures. 14 | #' @param \ldots Additional arguments passed 15 | #' to [BatchtoolsFuture()]. 16 | #' 17 | #' @return An object of class `BatchtoolsMulticoreFuture`. 18 | #' 19 | #' @details 20 | #' batchtools multicore futures rely on the batchtools backend set 21 | #' up by [batchtools::makeClusterFunctionsMulticore()]. 22 | #' The batchtools multicore backend only works on operating systems 23 | #' supporting the `ps` command-line tool, e.g. Linux and macOS. 24 | #' 25 | #' @importFrom batchtools makeClusterFunctionsMulticore 26 | #' @importFrom parallelly availableCores 27 | #' @export 28 | #' @keywords internal 29 | batchtools_multicore <- function(expr, envir = parent.frame(), 30 | substitute = TRUE, globals = TRUE, 31 | label = NULL, 32 | workers = availableCores(constraints = "multicore"), 33 | registry = list(), ...) { 34 | if (substitute) expr <- substitute(expr) 35 | 36 | if (is.null(workers)) { 37 | workers <- availableCores(constraints = "multicore") 38 | } else if (is.function(workers)) { 39 | workers <- workers() 40 | } 41 | stop_if_not(length(workers) == 1L, is.numeric(workers), 42 | is.finite(workers), workers >= 1L) 43 | 44 | ## Fall back to batchtools_local if multicore processing is not supported 45 | if ((workers == 1L && !inherits(workers, "AsIs")) || 46 | is_os("windows") || is_os("solaris") || 47 | availableCores(constraints = "multicore") == 1L) { 48 | ## covr: skip=1 49 | return(batchtools_local(expr, envir = envir, substitute = FALSE, 50 | globals = globals, label = label, 51 | registry = registry, ...)) 52 | } 53 | 54 | oopts <- options(mc.cores = workers) 55 | on.exit(options(oopts)) 56 | 57 | cf <- makeClusterFunctionsMulticore(ncpus = workers) 58 | 59 | future <- BatchtoolsMulticoreFuture( 60 | expr = expr, envir = envir, substitute = FALSE, 61 | globals = globals, 62 | label = label, 63 | cluster.functions = cf, 64 | registry = registry, 65 | ... 66 | ) 67 | 68 | if (!future$lazy) future <- run(future) 69 | 70 | invisible(future) 71 | } 72 | class(batchtools_multicore) <- c( 73 | "batchtools_multicore", "batchtools_multiprocess", "batchtools", 74 | "multiprocess", "future", "function" 75 | ) 76 | attr(batchtools_multicore, "tweakable") <- c("finalize") 77 | -------------------------------------------------------------------------------- /R/batchtools_ssh.R: -------------------------------------------------------------------------------- 1 | #' batchtools SSH futures 2 | #' 3 | #' A batchtools SSH future is an asynchronous multiprocess 4 | #' future that will be evaluated in a background R session.\cr 5 | #' \cr 6 | #' _We highly recommend using [future::multisession] 7 | #' (sic!) futures of the \pkg{future} package instead of 8 | #' SSH batchtools futures._ 9 | #' 10 | #' @inheritParams BatchtoolsFuture 11 | #' 12 | #' @param workers The number of SSH processes to be 13 | #' available for concurrent batchtools SSH futures. 14 | #' @param \ldots Additional arguments passed 15 | #' to [BatchtoolsFuture()]. 16 | #' 17 | #' @return An object of class `BatchtoolsMulticoreFuture`. 18 | #' 19 | #' @details 20 | #' batchtools SSH futures rely on the batchtools backend set 21 | #' up by [batchtools::makeClusterFunctionsSSH()]. 22 | #' The batchtools SSH backend only works on operating systems 23 | #' supporting the `ssh` and `ps` command-line tool, e.g. Linux and macOS. 24 | #' 25 | #' @importFrom parallelly availableWorkers 26 | #' 27 | #' @export 28 | #' @keywords internal 29 | batchtools_ssh <- function(expr, envir = parent.frame(), 30 | substitute = TRUE, globals = TRUE, 31 | label = NULL, 32 | workers = availableWorkers(), 33 | registry = list(), ...) { 34 | if (substitute) expr <- substitute(expr) 35 | 36 | future <- BatchtoolsSSHFuture( 37 | expr = expr, envir = envir, substitute = FALSE, 38 | globals = globals, 39 | label = label, 40 | workers = workers, 41 | registry = registry, 42 | ... 43 | ) 44 | 45 | if (!future$lazy) future <- run(future) 46 | 47 | invisible(future) 48 | } 49 | class(batchtools_ssh) <- c( 50 | "batchtools_ssh", "batchtools_custom", 51 | "batchtools_multiprocess", "batchtools", 52 | "multiprocess", "future", "function" 53 | ) 54 | attr(batchtools_ssh, "tweakable") <- c("finalize") 55 | -------------------------------------------------------------------------------- /R/batchtools_template.R: -------------------------------------------------------------------------------- 1 | #' Batchtools futures for LSF, OpenLava, SGE, Slurm, TORQUE etc. 2 | #' 3 | #' Batchtools futures for LSF, OpenLava, SGE, Slurm, TORQUE etc. are 4 | #' asynchronous multiprocess futures that will be evaluated on a compute 5 | #' cluster via a job scheduler. 6 | #' 7 | #' @inheritParams BatchtoolsFuture 8 | #' 9 | #' @param template (optional) A batchtools template file or a template string 10 | #' (in \pkg{brew} format). If not specified, it is left to the 11 | #' \pkg{batchtools} package to locate such file using its search rules. 12 | #' 13 | #' @param \ldots Additional arguments passed to [BatchtoolsFuture()]. 14 | #' 15 | #' @return An object of class `BatchtoolsFuture`. 16 | #' 17 | #' @details 18 | #' These type of batchtools futures rely on batchtools backends set 19 | #' up using the following \pkg{batchtools} functions: 20 | #' 21 | #' * [batchtools::makeClusterFunctionsLSF()] for 22 | #' [Load Sharing Facility (LSF)](https://en.wikipedia.org/wiki/Platform_LSF) 23 | #' * [batchtools::makeClusterFunctionsOpenLava()] for 24 | #' [OpenLava](https://en.wikipedia.org/wiki/OpenLava) 25 | #' * [batchtools::makeClusterFunctionsSGE()] for 26 | #' [Sun/Oracle Grid Engine (SGE)](https://en.wikipedia.org/wiki/Oracle_Grid_Engine) 27 | #' * [batchtools::makeClusterFunctionsSlurm()] for 28 | #' [Slurm](https://en.wikipedia.org/wiki/Slurm_Workload_Manager) 29 | #' * [batchtools::makeClusterFunctionsTORQUE()] for 30 | #' [TORQUE](https://en.wikipedia.org/wiki/TORQUE) / PBS 31 | #' 32 | #' @export 33 | #' @rdname batchtools_template 34 | #' @name batchtools_template 35 | batchtools_lsf <- function(expr, envir = parent.frame(), substitute = TRUE, 36 | globals = TRUE, label = NULL, 37 | template = NULL, resources = list(), 38 | workers = NULL, 39 | registry = list(), ...) { 40 | if (substitute) expr <- substitute(expr) 41 | 42 | batchtools_by_template(expr, envir = envir, substitute = FALSE, 43 | globals = globals, label = label, template = template, 44 | type = "lsf", resources = resources, 45 | workers = workers, registry = registry, ...) 46 | } 47 | class(batchtools_lsf) <- c( 48 | "batchtools_lsf", "batchtools_template", 49 | "batchtools_multiprocess", "batchtools", 50 | "multiprocess", "future", "function" 51 | ) 52 | attr(batchtools_lsf, "tweakable") <- c( 53 | "finalize", 54 | ## Arguments to batchtools::makeClusterFunctionsLSF() 55 | "scheduler.latency", "fs.latency" 56 | ) 57 | 58 | #' @export 59 | #' @rdname batchtools_template 60 | batchtools_openlava <- function(expr, envir = parent.frame(), substitute = TRUE, 61 | globals = TRUE, label = NULL, 62 | template = NULL, resources = list(), 63 | workers = NULL, 64 | registry = list(), ...) { 65 | if (substitute) expr <- substitute(expr) 66 | 67 | batchtools_by_template(expr, envir = envir, substitute = FALSE, 68 | globals = globals, label = label, template = template, 69 | type = "openlava", resources = resources, 70 | workers = workers, registry = registry, ...) 71 | } 72 | class(batchtools_openlava) <- c( 73 | "batchtools_openlava", "batchtools_template", 74 | "batchtools_multiprocess", "batchtools", 75 | "multiprocess", "future", "function" 76 | ) 77 | attr(batchtools_openlava, "tweakable") <- c( 78 | "finalize", 79 | ## Arguments to batchtools::makeClusterFunctionsOpenLava() 80 | "scheduler.latency", "fs.latency" 81 | ) 82 | 83 | #' @export 84 | #' @rdname batchtools_template 85 | batchtools_sge <- function(expr, envir = parent.frame(), substitute = TRUE, 86 | globals = TRUE, label = NULL, 87 | template = NULL, resources = list(), 88 | workers = NULL, 89 | registry = list(), ...) { 90 | if (substitute) expr <- substitute(expr) 91 | 92 | batchtools_by_template(expr, envir = envir, substitute = FALSE, 93 | globals = globals, label = label, template = template, 94 | type = "sge", resources = resources, 95 | workers = workers, registry = registry, ...) 96 | } 97 | class(batchtools_sge) <- c( 98 | "batchtools_sge", "batchtools_template", 99 | "batchtools_multiprocess", "batchtools", 100 | "multiprocess", "future", "function" 101 | ) 102 | attr(batchtools_sge, "tweakable") <- c( 103 | "finalize", 104 | ## Arguments to batchtools::makeClusterFunctionsSGE() 105 | "nodename", "scheduler.latency", "fs.latency" 106 | ) 107 | 108 | #' @export 109 | #' @rdname batchtools_template 110 | batchtools_slurm <- function(expr, envir = parent.frame(), substitute = TRUE, 111 | globals = TRUE, label = NULL, 112 | template = NULL, resources = list(), 113 | workers = NULL, 114 | registry = list(), ...) { 115 | if (substitute) expr <- substitute(expr) 116 | 117 | batchtools_by_template(expr, envir = envir, substitute = FALSE, 118 | globals = globals, label = label, template = template, 119 | type = "slurm", resources = resources, 120 | workers = workers, registry = registry, ...) 121 | } 122 | class(batchtools_slurm) <- c( 123 | "batchtools_slurm", "batchtools_template", 124 | "batchtools_multiprocess", "batchtools", 125 | "multiprocess", "future", "function" 126 | ) 127 | attr(batchtools_slurm, "tweakable") <- c( 128 | "finalize", 129 | ## Arguments to batchtools::makeClusterFunctionsSlurm() 130 | "array.jobs", "nodename", "scheduler.latency", "fs.latency" 131 | ) 132 | 133 | 134 | #' @export 135 | #' @rdname batchtools_template 136 | batchtools_torque <- function(expr, envir = parent.frame(), substitute = TRUE, 137 | globals = TRUE, label = NULL, 138 | template = NULL, resources = list(), 139 | workers = NULL, 140 | registry = list(), ...) { 141 | if (substitute) expr <- substitute(expr) 142 | 143 | batchtools_by_template(expr, envir = envir, substitute = FALSE, 144 | globals = globals, label = label, template = template, 145 | type = "torque", resources = resources, 146 | workers = workers, registry = registry, ...) 147 | } 148 | class(batchtools_torque) <- c( 149 | "batchtools_torque", "batchtools_template", 150 | "batchtools_multiprocess", "batchtools", 151 | "multiprocess", "future", "function" 152 | ) 153 | attr(batchtools_torque, "tweakable") <- c( 154 | "finalize", 155 | ## Arguments to batchtools::makeClusterFunctionsTORQUE() 156 | "scheduler.latency", "fs.latency" 157 | ) 158 | 159 | 160 | #' @importFrom batchtools makeClusterFunctionsLSF 161 | #' @importFrom batchtools makeClusterFunctionsOpenLava 162 | #' @importFrom batchtools makeClusterFunctionsSGE 163 | #' @importFrom batchtools makeClusterFunctionsSlurm 164 | #' @importFrom batchtools makeClusterFunctionsTORQUE 165 | batchtools_by_template <- function(expr, envir = parent.frame(), 166 | substitute = TRUE, globals = TRUE, 167 | template = NULL, 168 | type = c("lsf", "openlava", "sge", 169 | "slurm", "torque"), 170 | resources = list(), label = NULL, 171 | workers = NULL, 172 | registry = list(), ...) { 173 | if (substitute) expr <- substitute(expr) 174 | type <- match.arg(type) 175 | 176 | dotdotdot <- list(...) 177 | 178 | make_cfs <- switch(type, 179 | lsf = makeClusterFunctionsLSF, 180 | openlava = makeClusterFunctionsOpenLava, 181 | sge = makeClusterFunctionsSGE, 182 | slurm = makeClusterFunctionsSlurm, 183 | torque = makeClusterFunctionsTORQUE 184 | ) 185 | 186 | constructor <- switch(type, 187 | lsf = BatchtoolsLsfFuture, 188 | openlava = BatchtoolsOpenLavaFuture, 189 | sge = BatchtoolsSGEFuture, 190 | slurm = BatchtoolsSlurmFuture, 191 | torque = BatchtoolsTorqueFuture 192 | ) 193 | 194 | make_cfs_formals <- formals(make_cfs) 195 | 196 | ## Get the default template? 197 | if (is.null(template)) { 198 | template <- make_cfs_formals$template 199 | } 200 | 201 | stop_if_not(is.character(template), length(template) == 1L, 202 | !is.na(template), nzchar(template)) 203 | 204 | template <- find_template_file(template) 205 | 206 | keep <- which(names(dotdotdot) %in% names(make_cfs_formals)) 207 | args <- c(list(template = template), dotdotdot[keep]) 208 | cluster.functions <- do.call(make_cfs, args = args) 209 | attr(cluster.functions, "template") <- template 210 | 211 | ## Drop used '...' arguments 212 | if (length(keep) > 0) dotdotdot <- dotdotdot[-keep] 213 | 214 | args <- list( 215 | expr = quote(expr), ## Avoid 'expr' being resolved by do.call() 216 | substitute = FALSE, envir = envir, 217 | globals = globals, 218 | label = label, 219 | cluster.functions = cluster.functions, 220 | registry = registry, 221 | resources = resources, 222 | workers = workers 223 | ) 224 | if (length(dotdotdot) > 0) args <- c(args, dotdotdot) 225 | future <- do.call(constructor, args = args) 226 | 227 | if (!future$lazy) future <- run(future) 228 | 229 | invisible(future) 230 | } ## batchtools_by_template() 231 | -------------------------------------------------------------------------------- /R/find_template_file.R: -------------------------------------------------------------------------------- 1 | #' @importFrom batchtools findTemplateFile 2 | #' @importFrom utils file_test 3 | find_template_file <- function(template) { 4 | pathname <- findTemplateFile(template) 5 | if (is.na(pathname)) { 6 | pathname <- system.file("templates", sprintf("%s.tmpl", template), 7 | package = .packageName) 8 | if (!file_test("-f", pathname)) pathname <- NA_character_ 9 | } 10 | if (is.na(pathname)) { 11 | stopf("Failed to locate a batchtools template file: *%s.tmpl", template) 12 | } 13 | pathname 14 | } 15 | -------------------------------------------------------------------------------- /R/future.batchtools-package.R: -------------------------------------------------------------------------------- 1 | #' future.batchtools: A Future for batchtools 2 | #' 3 | #' The \pkg{future.batchtools} package implements the Future API 4 | #' on top of \pkg{batchtools} such that futures can be resolved 5 | #' on for instance high-performance compute (HPC) clusters via 6 | #' job schedulers. 7 | #' The Future API is defined by the \pkg{future} package. 8 | #' 9 | #' To use batchtools futures, load \pkg{future.batchtools}, and 10 | #' select the type of future you wish to use via 11 | #' [future::plan()]. 12 | #' 13 | #' @example incl/future.batchtools.R 14 | #' 15 | #' @examples 16 | #' \donttest{ 17 | #' plan(batchtools_local) 18 | #' demo("mandelbrot", package = "future", ask = FALSE) 19 | #' } 20 | #' 21 | #' @name future.batchtools 22 | #' @aliases future.batchtools-package 23 | "_PACKAGE" 24 | -------------------------------------------------------------------------------- /R/future_cache_path.R: -------------------------------------------------------------------------------- 1 | #' @importFrom utils file_test sessionInfo 2 | future_cache_path <- local({ 3 | ## The subfolder used for this session 4 | dir <- NULL 5 | 6 | function(root_path = getOption("future.cache.path", ".future"), absolute = TRUE, create = TRUE) { 7 | if (is.null(dir)) { 8 | id <- basename(tempdir()) 9 | id <- gsub("Rtmp", "", id, fixed = TRUE) 10 | timestamp <- format(Sys.time(), format = "%Y%m%d_%H%M%S") 11 | dir <<- sprintf("%s-%s", timestamp, id) 12 | } 13 | 14 | path <- file.path(root_path, dir) 15 | if (create && !file_test("-d", path)) { 16 | dir.create(path, recursive = TRUE) 17 | pathname <- file.path(path, "sessioninfo.txt") 18 | writeLines(capture_output(print(sessionInfo())), con = pathname) 19 | } 20 | 21 | if (absolute) path <- normalizePath(path, mustWork = FALSE) 22 | 23 | path 24 | } 25 | }) 26 | -------------------------------------------------------------------------------- /R/nbrOfWorkers.R: -------------------------------------------------------------------------------- 1 | #' Gets the number of batchtools workers 2 | #' 3 | #' Tries to infer the total number of batchtools workers. This is 4 | #' done using various ad-hoc procedures based on code inspection 5 | #' of batchtools itself. 6 | #' 7 | #' @param evaluator A future evaluator function. 8 | #' If NULL (default), the current evaluator as returned 9 | #' by [future::plan()] is used. 10 | #' 11 | #' @return A number in \eqn{[1, Inf]}. 12 | #' 13 | #' @importFrom future nbrOfWorkers 14 | #' @export 15 | #' @keywords internal 16 | nbrOfWorkers.batchtools <- function(evaluator) { 17 | ## 1. Infer from 'workers' argument 18 | expr <- formals(evaluator)$workers 19 | workers <- eval(expr, enclos = baseenv()) 20 | if (!is.null(workers)) { 21 | if (is.function(workers)) workers <- workers() 22 | stop_if_not(length(workers) >= 1) 23 | if (is.numeric(workers)) return(prod(workers)) 24 | if (is.character(workers)) return(length(workers)) 25 | stop("Invalid data type of 'workers': ", mode(workers)) 26 | } 27 | 28 | ## 2. Infer from 'cluster.functions' argument 29 | expr <- formals(evaluator)$cluster.functions 30 | cf <- eval(expr, enclos = baseenv()) 31 | if (!is.null(cf)) { 32 | stop_if_not(inherits(cf, "ClusterFunctions")) 33 | 34 | name <- cf$name 35 | if (is.null(name)) name <- cf$Name 36 | 37 | ## Uni-process backends 38 | if (name %in% c("Local", "Interactive")) return(1L) 39 | 40 | ## Cluster backends (with a scheduler queue) 41 | if (name %in% c("TORQUE", "Slurm", "SGE", "OpenLava", "LSF")) { 42 | return(availableHpcWorkers()) 43 | } 44 | } 45 | 46 | ## If still not known, assume a generic HPC scheduler 47 | availableHpcWorkers() 48 | } 49 | 50 | #' @export 51 | nbrOfWorkers.batchtools_uniprocess <- function(evaluator) { 52 | assert_no_positional_args_but_first() 53 | 1L 54 | } 55 | 56 | #' @export 57 | nbrOfWorkers.batchtools_multicore <- function(evaluator) { 58 | ## 1. Infer from 'workers' argument 59 | expr <- formals(evaluator)$workers 60 | workers <- eval(expr, enclos = baseenv()) 61 | if (is.function(workers)) workers <- workers() 62 | stop_if_not(length(workers) == 1L, is.numeric(workers), !is.na(workers), is.finite(workers), workers >= 1) 63 | workers 64 | } 65 | 66 | #' @importFrom future nbrOfWorkers nbrOfFreeWorkers 67 | #' @export 68 | nbrOfFreeWorkers.batchtools <- function(evaluator, background = FALSE, ...) { 69 | ## Special case #1: Fall back to uniprocess processing 70 | if (inherits(evaluator, "uniprocess")) { 71 | return(NextMethod()) 72 | } 73 | 74 | ## Special case #2: Infinite number of workers 75 | workers <- nbrOfWorkers(evaluator) 76 | if (is.infinite(workers)) return(workers) 77 | 78 | ## In all other cases, we need to figure out how many workers 79 | ## are running at the moment 80 | 81 | warnf("nbrOfFreeWorkers() for %s is not fully implemented. For now, it'll assume that none of the workers are occupied", setdiff(class(evaluator), c("FutureStrategy", "tweaked"))[1]) 82 | usedWorkers <- 0L ## Mockup for now 83 | 84 | workers <- workers - usedWorkers 85 | stop_if_not(length(workers) == 1L, !is.na(workers), workers >= 0L) 86 | workers 87 | } 88 | 89 | 90 | #' @export 91 | nbrOfFreeWorkers.batchtools_uniprocess <- function(evaluator, background = FALSE, ...) { 92 | assert_no_positional_args_but_first() 93 | if (isTRUE(background)) 0L else 1L 94 | } 95 | 96 | 97 | 98 | #' @export 99 | nbrOfFreeWorkers.batchtools_multiprocess <- function(evaluator, background = FALSE, ...) { 100 | assert_no_positional_args_but_first() 101 | 102 | workers <- nbrOfWorkers(evaluator) 103 | 104 | ## Create a dummy future 105 | future <- evaluator(NULL, globals = FALSE, lazy = TRUE) 106 | freg <- sprintf("workers-%s", class(future)[1]) 107 | usedWorkers <- length(FutureRegistry(freg, action = "list")) 108 | 109 | workers <- workers - usedWorkers 110 | stop_if_not(length(workers) == 1L, !is.na(workers), workers >= 0L) 111 | workers 112 | } 113 | 114 | 115 | 116 | ## Number of available workers in an HPC environment 117 | ## 118 | ## @return (numeric) A positive integer or `+Inf`. 119 | availableHpcWorkers <- function() { 120 | name <- "future.batchtools.workers" 121 | value <- getOption(name, default = 100) 122 | if (!is.numeric(value) || length(value) != 1L || 123 | is.na(value) || value < 1.0) { 124 | stopf("Option %s does not specify a value >= 1: %s", 125 | sQuote(name), paste(sQuote(value), collapse = ", ")) 126 | } 127 | value <- floor(value) 128 | value 129 | } 130 | -------------------------------------------------------------------------------- /R/options.R: -------------------------------------------------------------------------------- 1 | #' Options used for batchtools futures 2 | #' 3 | #' Below are the \R options and environment variables that are used by the 4 | #' \pkg{future.batchtools} package. 5 | #' See [future::future.options] for additional ones that apply to futures 6 | #' in general.\cr 7 | #' \cr 8 | #' _WARNING: Note that the names and the default values of these options 9 | #' may change in future versions of the package. Please use with care 10 | #' until further notice._ 11 | #' 12 | #' @section Settings for batchtools futures: 13 | #' \describe{ 14 | #' \item{\option{future.batchtools.workers}:}{(a positive numeric or `+Inf`) 15 | #' The default number of workers available on HPC schedulers with 16 | #' job queues. (Default: `100`)} 17 | #' 18 | #' \item{\option{future.batchtools.output}:}{(logical) 19 | #' If TRUE, \pkg{batchtools} will produce extra output. 20 | #' If FALSE, such output will be disabled by setting \pkg{batchtools} 21 | #' options \option{batchtools.verbose} and \option{batchtools.progress} 22 | #' to FALSE. 23 | #' (Default: `getOption("future.debug", FALSE)`)} 24 | #' 25 | #' \item{\option{future.batchtools.expiration.tail}:}{(a positive numeric) 26 | #' When a \pkg{batchtools} job expires, the last few lines will be 27 | #' relayed by batchtools futures to help troubleshooting. 28 | #' This option controls how many lines are displayed. 29 | #' (Default: `48L`)} 30 | #' 31 | #' \item{\option{future.cache.path}:}{ 32 | #' (character string) 33 | #' An absolute or relative path specifying the root folder in which 34 | #' \pkg{batchtools} registry folders are stored. 35 | #' This folder needs to be accessible from all hosts ("workers"). 36 | #' Specifically, it must _not_ be a folder that is only local to the 37 | #' machine such as `file.path(tempdir(), ".future"` if an job scheduler 38 | #' on a HPC environment is used. 39 | #' (Default: `.future` in the current working directory)} 40 | #' 41 | #' \item{\option{future.delete}:}{(logical) 42 | #' Controls whether or not the future's \pkg{batchtools} registry folder 43 | #' is deleted after the future result has been collected. 44 | #' If TRUE, it is always deleted. 45 | #' If FALSE, it is never deleted. 46 | #' If not set or NULL, the it is deleted, unless running in non-interactive 47 | #' mode and the batchtools job failed or expired, which helps to 48 | #' troubleshoot when running in batch mode. 49 | #' (Default: NULL (not set))} 50 | #' } 51 | #' 52 | #' @section Environment variables that set R options: 53 | #' All of the above \R \option{future.batchtools.*} options can be set by 54 | #' corresponding environment variable \env{R_FUTURE_BATCHTOOLS_*} _when 55 | #' the \pkg{future.batchtools} package is loaded_. This means that those 56 | #' environment variables must be set before the \pkg{future.batchtools} 57 | #' package is loaded in order to have an effect. 58 | #' For example, if `R_FUTURE_BATCHTOOLS_WORKERS="200"` is set, then option 59 | #' \option{future.batchtools.workers} is set to `200` (numeric). 60 | #' 61 | #' @examples 62 | #' # Set an R option: 63 | #' options(future.cache.path = "/cluster-wide/folder/.future") 64 | #' 65 | #' @aliases 66 | #' future.cache.path 67 | #' future.delete 68 | #' R_FUTURE_CACHE_PATH 69 | #' R_FUTURE_DELETE 70 | #' future.batchtools.expiration.tail 71 | #' future.batchtools.output 72 | #' future.batchtools.workers 73 | #' R_FUTURE_BATCHTOOLS_EXPIRATION_TAIL 74 | #' R_FUTURE_BATCHTOOLS_OUTPUT 75 | #' R_FUTURE_BATCHTOOLS_WORKERS 76 | #' 77 | #' @name future.batchtools.options 78 | NULL 79 | 80 | 81 | 82 | 83 | 84 | 85 | # Set an R option from an environment variable 86 | update_package_option <- function(name, mode = "character", default = NULL, split = NULL, trim = TRUE, disallow = c("NA"), force = FALSE, debug = FALSE) { 87 | ## Nothing to do? 88 | value <- getOption(name, NULL) 89 | if (!force && !is.null(value)) return(getOption(name, default = default)) 90 | 91 | ## name="future.plan.disallow" => env="R_FUTURE_PLAN_DISALLOW" 92 | env <- gsub(".", "_", toupper(name), fixed = TRUE) 93 | env <- paste("R_", env, sep = "") 94 | 95 | env_value <- value <- Sys.getenv(env, unset = NA_character_) 96 | ## Nothing to do? 97 | if (is.na(value)) { 98 | if (debug) mdebugf("Environment variable %s not set", sQuote(env)) 99 | return(getOption(name, default = default)) 100 | } 101 | 102 | if (debug) mdebugf("%s=%s", env, sQuote(value)) 103 | 104 | ## Trim? 105 | if (trim) value <- trim(value) 106 | 107 | ## Nothing to do? 108 | if (!nzchar(value)) return(getOption(name, default = default)) 109 | 110 | ## Split? 111 | if (!is.null(split)) { 112 | value <- strsplit(value, split = split, fixed = TRUE) 113 | value <- unlist(value, use.names = FALSE) 114 | if (trim) value <- trim(value) 115 | } 116 | 117 | ## Coerce? 118 | mode0 <- storage.mode(value) 119 | if (mode0 != mode) { 120 | suppressWarnings({ 121 | storage.mode(value) <- mode 122 | }) 123 | if (debug) { 124 | mdebugf("Coercing from %s to %s: %s", mode0, mode, commaq(value)) 125 | } 126 | } 127 | 128 | if (length(disallow) > 0) { 129 | if ("NA" %in% disallow) { 130 | if (any(is.na(value))) { 131 | stopf("Coercing environment variable %s=%s to %s would result in missing values for option %s: %s", sQuote(env), sQuote(env_value), sQuote(mode), sQuote(name), commaq(value)) 132 | } 133 | } 134 | if (is.numeric(value)) { 135 | if ("non-positive" %in% disallow) { 136 | if (any(value <= 0, na.rm = TRUE)) { 137 | stopf("Environment variable %s=%s specifies a non-positive value for option %s: %s", sQuote(env), sQuote(env_value), sQuote(name), commaq(value)) 138 | } 139 | } 140 | if ("negative" %in% disallow) { 141 | if (any(value < 0, na.rm = TRUE)) { 142 | stopf("Environment variable %s=%s specifies a negative value for option %s: %s", sQuote(env), sQuote(env_value), sQuote(name), commaq(value)) 143 | } 144 | } 145 | } 146 | } 147 | 148 | if (debug) { 149 | mdebugf("=> options(%s = %s) [n=%d, mode=%s]", 150 | dQuote(name), commaq(value), 151 | length(value), storage.mode(value)) 152 | } 153 | 154 | do.call(options, args = structure(list(value), names = name)) 155 | 156 | getOption(name, default = default) 157 | } 158 | 159 | 160 | ## Set future options based on environment variables 161 | update_package_options <- function(debug = FALSE) { 162 | update_package_option("future.cache.path", mode = "character", debug = debug) 163 | update_package_option("future.delete", mode = "logical", debug = debug) 164 | 165 | update_package_option("future.batchtools.expiration.tail", mode = "integer", debug = debug) 166 | update_package_option("future.batchtools.output", mode = "logical", debug = debug) 167 | update_package_option("future.batchtools.workers", mode = "numeric", debug = debug) 168 | update_package_option("future.batchtools.status.cache", mode = "logical", default = TRUE, debug = debug) 169 | } 170 | -------------------------------------------------------------------------------- /R/resources_OP.R: -------------------------------------------------------------------------------- 1 | #' Temporarily tweaks the resources for the current batchtools strategy 2 | #' 3 | #' @usage fassignment \%resources\% tweaks 4 | #' 5 | #' @param fassignment The future assignment, e.g. 6 | #' \code{x \%<-\% \{ expr \}}. 7 | #' @param tweaks A named list (or vector) of resource \pkg{batchtools} 8 | #' parameters (see Section 'Resources' in [batchtools::submitJobs()]) 9 | #' that should be changed relative to the current strategy. 10 | #' 11 | #' @export 12 | #' @importFrom future plan tweak 13 | #' @keywords internal 14 | `%resources%` <- function(fassignment, tweaks) { 15 | fassignment <- substitute(fassignment) 16 | envir <- parent.frame(1) 17 | stop_if_not(is.vector(tweaks)) 18 | tweaks <- as.list(tweaks) 19 | stop_if_not(!is.null(names(tweaks))) 20 | 21 | ## Temporarily use a different plan 22 | oplan <- plan("list") 23 | on.exit(plan(oplan, substitute = FALSE, .call = NULL)) 24 | 25 | ## Tweak current strategy and apply 26 | args <- list(plan("next"), resources = tweaks, penvir = envir) 27 | strategy <- do.call(tweak, args = args) 28 | plan(strategy, substitute = FALSE, .call = NULL) 29 | 30 | eval(fassignment, envir = envir, enclos = baseenv()) 31 | } 32 | -------------------------------------------------------------------------------- /R/temp_registry.R: -------------------------------------------------------------------------------- 1 | #' @importFrom batchtools makeRegistry saveRegistry 2 | temp_registry <- local({ 3 | ## All known batchtools registries 4 | regs <- new.env() 5 | 6 | make_registry <- function(cluster.functions = NULL, config = list(), ...) { 7 | ## Temporarily disable batchtools output? 8 | ## (i.e. messages and progress bars) 9 | debug <- isTRUE(getOption("future.debug")) 10 | batchtools_output <- getOption("future.batchtools.output", debug) 11 | 12 | work.dir <- config$work.dir 13 | if (is.null(work.dir)) work.dir <- getwd() 14 | config$work.dir <- NULL 15 | 16 | if (!batchtools_output) { 17 | oopts <- options(batchtools.verbose = FALSE, batchtools.progress = FALSE) 18 | on.exit(options(oopts)) 19 | } 20 | 21 | ## WORKAROUND: batchtools::makeRegistry() updates the RNG state, 22 | ## which we must make sure to undo. 23 | with_stealth_rng({ 24 | reg <- makeRegistry(work.dir = work.dir, ...) 25 | }) 26 | 27 | if (!is.null(cluster.functions)) { ### FIXME 28 | reg$cluster.functions <- cluster.functions 29 | } 30 | 31 | ## Post-tweak the batchtools registry? 32 | ## This avoids having to set up a custom batchtools 'conf.file' etc. 33 | if (length(config) > 0L) { 34 | names <- names(config) 35 | for (name in names) reg[[name]] <- config[[name]] 36 | with_stealth_rng({ 37 | saveRegistry(reg) 38 | }) 39 | } 40 | 41 | reg 42 | } ## make_registry() 43 | 44 | function(label = "batchtools", path = NULL, config = list(), ...) { 45 | if (is.null(label)) label <- "batchtools" 46 | ## The job label (the name on the job queue) - may be duplicated 47 | label <- as.character(label) 48 | stop_if_not(length(label) == 1L, nchar(label) > 0L) 49 | 50 | ## This session's path holding all of its future batchtools directories 51 | ## e.g. .future/-/ 52 | if (is.null(path)) path <- future_cache_path() 53 | 54 | if (length(config) > 0L) { 55 | stop_if_not(is.list(config)) 56 | names <- names(config) 57 | stop_if_not(!is.null(names), all(nzchar(names))) 58 | } 59 | 60 | ## The batchtools subfolder for a specific future - must be unique 61 | prefix <- sprintf("%s_", label) 62 | 63 | ## FIXME: We need to make sure 'prefix' consists of only valid 64 | ## filename characters. /HB 2016-10-19 65 | prefix <- as_valid_directory_prefix(prefix) 66 | 67 | ## WORKAROUND: Avoid updating the RNG state 68 | with_stealth_rng({ 69 | unique <- FALSE 70 | while (!unique) { 71 | ## The FutureRegistry key for this batchtools future - must be unique 72 | key <- tempvar(prefix = prefix, value = NA, envir = regs) 73 | ## The directory for this batchtools future 74 | ## e.g. .future/-// 75 | path_registry <- file.path(path, key) 76 | ## Should not happen, but just in case. 77 | unique <- !file.exists(path_registry) 78 | } 79 | }) 80 | 81 | ## FIXME: We need to make sure 'label' consists of only valid 82 | ## batchtools ID characters, i.e. it must match regular 83 | ## expression "^[a-zA-Z]+[0-9a-zA-Z_]*$". 84 | ## /HB 2016-10-19 85 | reg_id <- as_valid_registry_id(label) 86 | make_registry(file.dir = path_registry, config = config, ...) 87 | } 88 | }) 89 | 90 | 91 | 92 | drop_non_valid_characters <- function(name, pattern, default = "batchtools") { 93 | as_string <- (length(name) == 1L) 94 | name <- unlist(strsplit(name, split = "", fixed = TRUE), use.names = FALSE) 95 | name[!grepl(pattern, name)] <- "" 96 | if (length(name) == 0L) return(default) 97 | if (as_string) name <- paste(name, collapse = "") 98 | name 99 | } 100 | 101 | as_valid_directory_prefix <- function(name) { 102 | pattern <- "^[-._a-zA-Z0-9]+$" 103 | ## Nothing to do? 104 | if (grepl(pattern, name)) return(name) 105 | name <- unlist(strsplit(name, split = "", fixed = TRUE), use.names = FALSE) 106 | ## All characters must be letters, digits, underscores, dash, or period. 107 | name <- drop_non_valid_characters(name, pattern = pattern) 108 | name <- paste(name, collapse = "") 109 | stop_if_not(grepl(pattern, name)) 110 | name 111 | } 112 | 113 | as_valid_registry_id <- function(name) { 114 | pattern <- "^[a-zA-Z]+[0-9a-zA-Z_]*$" 115 | ## Nothing to do? 116 | if (grepl(pattern, name)) return(name) 117 | 118 | name <- unlist(strsplit(name, split = "", fixed = TRUE), use.names = FALSE) 119 | 120 | ## All characters must be letters, digits, or underscores 121 | name <- drop_non_valid_characters(name, pattern = "[0-9a-zA-Z_]") 122 | name <- name[nzchar(name)] 123 | 124 | ## First character must be a letter :/ 125 | if (!grepl("^[a-zA-Z]+", name[1])) name[1] <- "z" 126 | 127 | name <- paste(name, collapse = "") 128 | 129 | stop_if_not(grepl(pattern, name)) 130 | 131 | name 132 | } 133 | -------------------------------------------------------------------------------- /R/utils,conditions.R: -------------------------------------------------------------------------------- 1 | stopf <- function(fmt, ..., call. = TRUE, domain = NULL) { #nolint 2 | msg <- sprintf(fmt, ...) 3 | msg <- .makeMessage(msg, domain = domain) 4 | if (is.call(call.)) { 5 | call <- call. 6 | } else if (isTRUE(call)) { 7 | call <- sys.call(which = -1L) 8 | } else { 9 | call <- NULL 10 | } 11 | cond <- simpleError(msg, call = call) 12 | stop(cond) 13 | } 14 | 15 | warnf <- function(fmt, ..., call. = TRUE, immediate. = FALSE, domain = NULL) { #nolint 16 | msg <- sprintf(fmt, ...) 17 | ## Cannot tweak 'call' when immediate. = TRUE 18 | if (isTRUE(immediate.)) { 19 | warning(msg, call. = call., immediate. = immediate., domain = domain) 20 | } else { 21 | msg <- .makeMessage(msg, domain = domain) 22 | if (is.call(call.)) { 23 | call <- call. 24 | } else if (isTRUE(call)) { 25 | call <- sys.call(which = -1L) 26 | } else { 27 | call <- NULL 28 | } 29 | cond <- simpleWarning(msg, call = call) 30 | warning(cond) 31 | } 32 | } 33 | 34 | msgf <- function(fmt, ..., appendLF = FALSE, domain = NULL) { #nolint 35 | message(sprintf(fmt, ...), appendLF = appendLF, domain = domain) 36 | } 37 | -------------------------------------------------------------------------------- /R/utils-debug.R: -------------------------------------------------------------------------------- 1 | now <- function(x = Sys.time(), format = "[%H:%M:%OS3] ") { 2 | ## format(x, format = format) ## slower 3 | format(as.POSIXlt(x, tz = ""), format = format) 4 | } 5 | 6 | debug_indent <- local({ 7 | symbols <- rep(c("|", ":", ".", "'", ",", ";", "`"), times = 10L) 8 | function() { 9 | depth <- length(.debug[["stack"]]) 10 | if (depth == 0) return("") 11 | indent <- getOption("future.debug.indent", " ") 12 | paste(paste(symbols[seq_len(depth)], indent, sep = ""), collapse = "") 13 | } 14 | }) 15 | 16 | if (!exists(".debug", inherits = FALSE)) .debug <- new.env(parent = emptyenv()) 17 | if (!"stack" %in% names(".debug")) .debug$stack <- list() 18 | 19 | mdebug_push <- function(...) { 20 | msg <- mdebug(...) 21 | .debug$stack <- c(.debug$stack, msg) 22 | invisible(msg) 23 | } 24 | 25 | mdebugf_push <- function(...) { 26 | msg <- mdebugf(...) 27 | .debug$stack <- c(.debug$stack, msg) 28 | invisible(msg) 29 | } 30 | 31 | mdebug_pop <- function(...) { 32 | n <- length(.debug$stack) 33 | msg <- c(...) 34 | if (length(msg) == 0) { 35 | msg <- .debug$stack[n] 36 | msg <- sprintf("%s done", msg) 37 | } 38 | .debug$stack <- .debug$stack[-n] 39 | if (length(msg) == 0 || !is.na(msg)) mdebug(msg) 40 | } 41 | 42 | mdebugf_pop <- function(...) { 43 | n <- length(.debug$stack) 44 | msg <- .debug$stack[n] 45 | .debug$stack <- .debug$stack[-n] 46 | mdebug(sprintf("%s done", msg)) 47 | } 48 | 49 | mdebug <- function(..., prefix = now()) { 50 | prefix <- paste(prefix, debug_indent(), sep = "") 51 | msg <- paste(..., sep = "") 52 | message(sprintf("%s%s", prefix, msg)) 53 | invisible(msg) 54 | } 55 | 56 | mdebugf <- function(..., appendLF = TRUE, prefix = now()) { 57 | prefix <- paste(prefix, debug_indent(), sep = "") 58 | msg <- sprintf(...) 59 | message(sprintf("%s%s", prefix, msg), appendLF = appendLF) 60 | invisible(msg) 61 | } 62 | 63 | #' @importFrom utils capture.output 64 | mprint <- function(..., appendLF = TRUE, prefix = now()) { 65 | prefix <- paste(prefix, debug_indent(), sep = "") 66 | message(paste(prefix, capture.output(print(...)), sep = "", collapse = "\n"), appendLF = appendLF) 67 | } 68 | 69 | #' @importFrom utils capture.output str 70 | mstr <- function(..., appendLF = TRUE, prefix = now()) { 71 | prefix <- paste(prefix, debug_indent(), sep = "") 72 | message(paste(prefix, capture.output(str(...)), sep = "", collapse = "\n"), appendLF = appendLF) 73 | } 74 | -------------------------------------------------------------------------------- /R/utils.R: -------------------------------------------------------------------------------- 1 | is_na <- function(x) { 2 | if (length(x) != 1L) return(FALSE) 3 | is.na(x) 4 | } 5 | 6 | is_false <- function(x) { 7 | if (length(x) != 1L) return(FALSE) 8 | x <- as.logical(x) 9 | x <- unclass(x) 10 | identical(FALSE, x) 11 | } 12 | 13 | stop_if_not <- function(...) { 14 | res <- list(...) 15 | for (ii in 1L:length(res)) { 16 | res_ii <- .subset2(res, ii) 17 | if (length(res_ii) != 1L || is.na(res_ii) || !res_ii) { 18 | mc <- match.call() 19 | call <- deparse(mc[[ii + 1]], width.cutoff = 60L) 20 | if (length(call) > 1L) call <- paste(call[1L], "....") 21 | stopf("%s is not TRUE", sQuote(call), call. = FALSE, domain = NA) 22 | } 23 | } 24 | 25 | NULL 26 | } 27 | 28 | attached_packages <- function() { 29 | pkgs <- search() 30 | pkgs <- grep("^package:", pkgs, value = TRUE) 31 | pkgs <- gsub("^package:", "", pkgs) 32 | pkgs 33 | } 34 | 35 | ## Adopted R.utils 2.1.0 (2015-06-15) 36 | #' @importFrom utils capture.output 37 | capture_output <- function(expr, envir = parent.frame(), ...) { 38 | res <- eval({ 39 | file <- rawConnection(raw(0L), open = "w") 40 | on.exit(close(file)) 41 | capture.output(expr, file = file) 42 | rawToChar(rawConnectionValue(file)) 43 | }, envir = envir, enclos = baseenv()) 44 | unlist(strsplit(res, split = "\n", fixed = TRUE), use.names = FALSE) 45 | } 46 | 47 | printf <- function(...) cat(sprintf(...)) 48 | 49 | ## From R.utils 2.0.2 (2015-05-23) 50 | hpaste <- function(..., sep="", collapse=", ", last_collapse=NULL, 51 | max_head=if (missing(last_collapse)) 3 else Inf, 52 | max_tail=if (is.finite(max_head)) 1 else Inf, 53 | abbreviate="...") { 54 | max_head <- as.double(max_head) 55 | max_tail <- as.double(max_tail) 56 | if (is.null(last_collapse)) last_collapse <- collapse 57 | 58 | # Build vector 'x' 59 | x <- paste(..., sep = sep) 60 | n <- length(x) 61 | 62 | # Nothing todo? 63 | if (n == 0) return(x) 64 | if (is.null(collapse)) return(x) 65 | 66 | # Abbreviate? 67 | if (n > max_head + max_tail + 1) { 68 | head <- x[seq_len(max_head)] 69 | tail <- rev(rev(x)[seq_len(max_tail)]) 70 | x <- c(head, abbreviate, tail) 71 | n <- length(x) 72 | } 73 | 74 | if (!is.null(collapse) && n > 1) { 75 | if (last_collapse == collapse) { 76 | x <- paste(x, collapse = collapse) 77 | } else { 78 | x_head <- paste(x[1:(n - 1)], collapse = collapse) 79 | x <- paste(x_head, x[n], sep = last_collapse) 80 | } 81 | } 82 | 83 | x 84 | } 85 | 86 | ## Adopted from R.oo 1.19.0 (2015-06-15) 87 | trim <- function(x, ...) { 88 | sub("[\t\n\f\r ]*$", "", sub("^[\t\n\f\r ]*", "", x)) 89 | } 90 | 91 | comma <- function(x, sep = ", ") paste(x, collapse = sep) 92 | 93 | commaq <- function(x, sep = ", ") paste(sQuote(x), collapse = sep) 94 | 95 | ## Evaluates an expression in global environment. 96 | ## Because geval() is exported, we want to keep its environment() 97 | ## as small as possible, which is why we use local(). Without, 98 | ## the environment would be that of the package itself and all of 99 | ## the package would be exported. 100 | geval <- local(function(expr, substitute = FALSE, envir = .GlobalEnv, enclos = baseenv(), ...) { 101 | if (substitute) expr <- substitute(expr) 102 | eval(expr, envir = envir, enclos = enclos) 103 | }) 104 | 105 | 106 | ## Tests if the current OS is of a certain type 107 | is_os <- function(name) { 108 | if (name == "windows") { 109 | return(.Platform$OS.type == "windows") 110 | } else { 111 | grepl(paste0("^", name), R.version$os) 112 | } 113 | } 114 | 115 | 116 | ## From R.utils 2.5.0 117 | tempvar <- function(prefix = "var", value = NA, envir = parent.frame()) { 118 | max_tries <- 1e6 119 | max_int <- .Machine$integer.max 120 | 121 | ii <- 0L 122 | while (ii < max_tries) { 123 | # Generate random variable name 124 | idx <- sample.int(max_int, size = 1L) 125 | name <- sprintf("%s%d", prefix, idx) 126 | 127 | # Available? 128 | if (!exists(name, envir = envir, inherits = FALSE)) { 129 | assign(name, value, envir = envir, inherits = FALSE) 130 | return(name) 131 | } 132 | 133 | ii <- ii + 1L 134 | } 135 | 136 | # Failed to find a unique temporary variable name 137 | stopf("Failed to generate a unique non-existing temporary variable with prefix '%s'", prefix) #nolint 138 | } 139 | 140 | 141 | 142 | result_has_errors <- function(result) { 143 | stop_if_not(inherits(result, "FutureResult")) 144 | 145 | for (c in result$conditions) { 146 | if (inherits(c$condition, "error")) return(TRUE) 147 | } 148 | 149 | FALSE 150 | } 151 | 152 | 153 | 154 | #' @importFrom utils file_test 155 | file_info <- function(file) { 156 | if (is.null(file) || is.na(file)) return("") 157 | if (file_test("-f", file)) { 158 | info <- sprintf("%d bytes", file.info(file)$size) 159 | n <- length(readLines(file, warn = FALSE)) 160 | info <- sprintf("%s; %d lines", info, n) 161 | } else { 162 | info <- "" 163 | } 164 | sprintf("%s (%s)", sQuote(file), info) 165 | } 166 | 167 | 168 | assert_no_positional_args_but_first <- function(call = sys.call(sys.parent())) { 169 | ast <- as.list(call) 170 | if (length(ast) <= 2L) return() 171 | ast <- ast[-(1:2)] 172 | dots <- vapply(ast, FUN = identical, as.symbol("..."), FUN.VALUE = FALSE) 173 | ast <- ast[!dots] 174 | if (length(ast) == 0L) return() 175 | names <- names(ast) 176 | if (is.null(names) || any(names == "")) { 177 | stopf("Function %s() requires that all arguments beyond the first one are passed by name and not by position: %s", as.character(call[[1L]]), deparse(call, width.cutoff = 100L)) 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /R/waitForWorker.R: -------------------------------------------------------------------------------- 1 | waitForWorker <- function(future, ...) { 2 | UseMethod("waitForWorker") 3 | } 4 | 5 | #' @export 6 | waitForWorker.default <- function(future, ...) NULL 7 | 8 | #' @export 9 | waitForWorker.BatchtoolsUniprocessFuture <- function(future, ...) NULL 10 | 11 | 12 | registerFuture <- function(future, ...) { 13 | UseMethod("registerFuture") 14 | } 15 | 16 | #' @export 17 | registerFuture.default <- function(future, ...) NULL 18 | 19 | #' @export 20 | registerFuture.BatchtoolsUniprocessFuture <- function(future, ...) NULL 21 | 22 | 23 | 24 | unregisterFuture <- function(future, ...) { 25 | UseMethod("unregisterFuture") 26 | } 27 | 28 | #' @export 29 | unregisterFuture.default <- function(future, ...) NULL 30 | 31 | #' @export 32 | unregisterFuture.BatchtoolsUniprocessFuture <- function(future, ...) NULL 33 | 34 | 35 | #' @export 36 | registerFuture.BatchtoolsFuture <- function(future, ...) { 37 | freg <- sprintf("workers-%s", class(future)[1]) 38 | FutureRegistry(freg, action = "add", future = future, earlySignal = FALSE, ...) 39 | } 40 | 41 | 42 | #' @export 43 | unregisterFuture.BatchtoolsFuture <- function(future, ...) { 44 | freg <- sprintf("workers-%s", class(future)[1]) 45 | FutureRegistry(freg, action = "remove", future = future, ...) 46 | } 47 | 48 | 49 | #' @importFrom future FutureError 50 | #' @export 51 | waitForWorker.BatchtoolsFuture <- function(future, 52 | workers, 53 | await = NULL, 54 | timeout = getOption("future.wait.timeout", 30 * 24 * 60 * 60), 55 | delta = getOption("future.wait.interval", 0.2), 56 | alpha = getOption("future.wait.alpha", 1.01), 57 | ...) { 58 | debug <- isTRUE(getOption("future.debug")) 59 | if (debug) { 60 | mdebugf_push("waitForWorker() for %s ...", class(future)[1]) 61 | on.exit(mdebug_pop()) 62 | } 63 | 64 | stop_if_not(is.null(await) || is.function(await)) 65 | workers <- as.integer(workers) 66 | stop_if_not(length(workers) == 1, is.finite(workers), workers >= 1L) 67 | stop_if_not(length(timeout) == 1, is.finite(timeout), timeout >= 0) 68 | stop_if_not(length(alpha) == 1, is.finite(alpha), alpha > 0) 69 | 70 | freg <- sprintf("workers-%s", class(future)[1]) 71 | 72 | ## Use a default await() function? 73 | if (is.null(await)) { 74 | await <- function() FutureRegistry(freg, action = "collect-first") 75 | } 76 | 77 | ## Number of occupied workers 78 | usedWorkers <- function() { 79 | length(FutureRegistry(freg, action = "list", earlySignal = FALSE)) 80 | } 81 | 82 | t0 <- Sys.time() 83 | dt <- 0 84 | iter <- 1L 85 | interval <- delta 86 | finished <- FALSE 87 | while (dt <= timeout) { 88 | ## Check for available workers 89 | used <- usedWorkers() 90 | finished <- (used < workers) 91 | if (finished) break 92 | 93 | if (debug) mdebugf("Poll #%d (%s): usedWorkers() = %d, workers = %d", iter, format(round(dt, digits = 2L)), used, workers) 94 | 95 | ## Wait 96 | Sys.sleep(interval) 97 | interval <- alpha * interval 98 | 99 | ## Finish/close workers, iff possible 100 | await() 101 | 102 | iter <- iter + 1L 103 | dt <- difftime(Sys.time(), t0) 104 | } 105 | 106 | if (!finished) { 107 | msg <- sprintf("TIMEOUT: All %d workers are still occupied after %s (polled %d times)", workers, format(round(dt, digits = 2L)), iter) 108 | if (debug) mdebug(msg) 109 | stop(FutureError(msg)) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /R/with_stealth_rng.R: -------------------------------------------------------------------------------- 1 | ## Evaluates an R expression while preventing any changes to .Random.seed 2 | with_stealth_rng <- function(expr, substitute = TRUE, envir = parent.frame(), ...) { 3 | if (substitute) expr <- substitute(expr) 4 | 5 | ## Record the original RNG state 6 | oseed <- .GlobalEnv$.Random.seed 7 | on.exit({ 8 | if (is.null(oseed)) { 9 | if (exists(".Random.seed", envir = .GlobalEnv, inherits = FALSE)) { 10 | rm(list = ".Random.seed", envir = .GlobalEnv, inherits = FALSE) 11 | } 12 | } else { 13 | .GlobalEnv$.Random.seed <- oseed 14 | } 15 | }) 16 | 17 | ## Evaluate the R expression with "random" RNG state 18 | if (!is.null(oseed)) { 19 | rm(list = ".Random.seed", envir = .GlobalEnv, inherits = FALSE) 20 | } 21 | eval(expr, envir = envir, enclos = baseenv()) 22 | } 23 | -------------------------------------------------------------------------------- /R/zzz.R: -------------------------------------------------------------------------------- 1 | ## To be cached by .onLoad() 2 | .onLoad <- function(libname, pkgname) { 3 | import_future_functions() 4 | 5 | debug <- isTRUE(getOption("future.debug")) 6 | 7 | inRCmdCheck <- import_parallelly("inRCmdCheck") 8 | if (inRCmdCheck()) { 9 | ## Don't write to current working directory when running R CMD check. 10 | path <- Sys.getenv("R_FUTURE_CACHE_PATH", NA_character_) 11 | if (is.na(path)) { 12 | Sys.setenv("R_FUTURE_CACHE_PATH" = file.path(tempdir(), ".future")) 13 | if (debug) { 14 | mdebugf("R CMD check detected: Set R_FUTURE_CACHE_PATH=%s", 15 | sQuote(Sys.getenv("R_FUTURE_CACHE_PATH"))) 16 | } 17 | } 18 | } 19 | 20 | update_package_options(debug = debug) 21 | } 22 | 23 | 24 | #' @importFrom utils file_test 25 | .onUnload <- function(libpath) { 26 | ## (a) Force finalizer of Future objects to run such 27 | ## that their batchtools directories are removed 28 | gc() 29 | 30 | ## (b) Remove batchtools root directory if only a set 31 | ## of known files exists, i.e. not any directories etc. 32 | path <- future_cache_path(create = FALSE) 33 | ## Only known files left? 34 | files <- dir(path = path) 35 | known_files <- c("sessioninfo.txt") 36 | if (all(files %in% known_files)) { 37 | for (file in known_files) { 38 | pathname_tmp <- file.path(path, file) 39 | if (file_test("-f", pathname_tmp)) try(file.remove(pathname_tmp)) 40 | } 41 | try(unlink(path, recursive = FALSE, force = TRUE), silent = TRUE) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /cran-comments.md: -------------------------------------------------------------------------------- 1 | # CRAN submission future.batchtools 0.12.1 2 | 3 | on 2023-12-19 4 | 5 | I've verified this submission has no negative impact on any of the 5 reverse package dependencies available on CRAN. 6 | 7 | Thank you 8 | 9 | 10 | ## Notes not sent to CRAN 11 | 12 | ### R CMD check validation 13 | 14 | The package has been verified using `R CMD check --as-cran` on: 15 | 16 | | R version | GitHub | R-hub | mac/win-builder | 17 | | --------- | ------ | ----- | --------------- | 18 | | 3.6.x | L | | | 19 | | 4.1.x | L | | | 20 | | 4.2.x | L M W | | | 21 | | 4.3.x | L M W | . W | M1 W | 22 | | devel | L M W | . | W | 23 | 24 | _Legend: OS: L = Linux, M = macOS, M1 = macOS M1, W = Windows_ 25 | 26 | 27 | R-hub checks: 28 | 29 | ```r 30 | res <- rhub::check(platforms = c( 31 | "debian-clang-devel", 32 | "fedora-gcc-devel", 33 | "debian-gcc-patched", 34 | "windows-x86_64-release" 35 | )) 36 | print(res) 37 | ``` 38 | 39 | gives 40 | 41 | ``` 42 | ── future.batchtools 0.12.1: OK 43 | 44 | Build ID: future.batchtools_0.12.1.tar.gz-0cf2af4123d14ff3ac742eb611644044 45 | Platform: Debian Linux, R-devel, clang, ISO-8859-15 locale 46 | Submitted: 3h 40m 16.2s ago 47 | Build time: 1h 10m 30.4s 48 | 49 | 0 errors ✔ | 0 warnings ✔ | 0 notes ✔ 50 | 51 | ── future.batchtools 0.12.1: OK 52 | 53 | Build ID: future.batchtools_0.12.1.tar.gz-f68417d7155c4af6a5ac7d6619aff851 54 | Platform: Fedora Linux, R-devel, GCC 55 | Submitted: 3h 40m 16.2s ago 56 | Build time: 52m 37.8s 57 | 58 | 0 errors ✔ | 0 warnings ✔ | 0 notes ✔ 59 | 60 | ── future.batchtools 0.12.1: OK 61 | 62 | Build ID: future.batchtools_0.12.1.tar.gz-26c0209f468c4dd5af5f3f4f908cb3c7 63 | Platform: Debian Linux, R-patched, GCC 64 | Submitted: 3h 40m 16.2s ago 65 | Build time: 1h 5m 2.8s 66 | 67 | 0 errors ✔ | 0 warnings ✔ | 0 notes ✔ 68 | 69 | ── future.batchtools 0.12.1: OK 70 | 71 | Build ID: future.batchtools_0.12.1.tar.gz-64ca57c8a5b543e68b5122ba5eb2a3d1 72 | Platform: Windows Server 2022, R-release, 32/64 bit 73 | Submitted: 3h 40m 16.2s ago 74 | Build time: 6m 41s 75 | 76 | 0 errors ✔ | 0 warnings ✔ | 0 notes ✔ 77 | ``` 78 | -------------------------------------------------------------------------------- /demo/00Index: -------------------------------------------------------------------------------- 1 | mandelbrot Mandelbrot set images using futures 2 | 3 | -------------------------------------------------------------------------------- /demo/mandelbrot.R: -------------------------------------------------------------------------------- 1 | library("future.batchtools") 2 | demo("mandelbrot", package = "future", ask = FALSE) 3 | message("\n\nTIPS: Try this demo with for instance plan(batchtools_local).\n") 4 | -------------------------------------------------------------------------------- /incl/batchtools_custom.R: -------------------------------------------------------------------------------- 1 | options(error = function(...) { 2 | print(traceback()) 3 | }) 4 | 5 | cf <- batchtools::makeClusterFunctionsInteractive(external = TRUE) 6 | print(cf) 7 | str(cf) 8 | plan(batchtools_custom, cluster.functions = cf) 9 | print(plan()) 10 | print(nbrOfWorkers()) 11 | 12 | ## Create explicit future 13 | f <- future({ 14 | cat("PID:", Sys.getpid(), "\n") 15 | 42L 16 | }) 17 | print(f) 18 | v <- value(f) 19 | print(v) 20 | 21 | options(error = NULL) 22 | 23 | 24 | ## Create explicit future 25 | f <- future({ 26 | cat("PID:", Sys.getpid(), "\n") 27 | 42L 28 | }) 29 | print(f) 30 | v <- value(f) 31 | print(v) 32 | 33 | 34 | 35 | ## Create explicit future 36 | f <- future({ 37 | cat("PID:", Sys.getpid(), "\n") 38 | 42L 39 | }) 40 | v <- value(f) 41 | print(v) 42 | -------------------------------------------------------------------------------- /incl/batchtools_local.R: -------------------------------------------------------------------------------- 1 | ## Use local batchtools futures 2 | plan(batchtools_local) 3 | 4 | ## A global variable 5 | a <- 1 6 | 7 | ## Create explicit future 8 | f <- future({ 9 | b <- 3 10 | c <- 2 11 | a * b * c 12 | }) 13 | v <- value(f) 14 | print(v) 15 | 16 | 17 | ## Create implicit future 18 | v %<-% { 19 | b <- 3 20 | c <- 2 21 | a * b * c 22 | } 23 | print(v) 24 | -------------------------------------------------------------------------------- /incl/future.batchtools.R: -------------------------------------------------------------------------------- 1 | library(future.batchtools) 2 | 3 | ## Use local batchtools futures 4 | plan(batchtools_local) 5 | 6 | ## A global variable 7 | a <- 1 8 | 9 | v %<-% { 10 | b <- 3 11 | c <- 2 12 | a * b * c 13 | } 14 | 15 | print(v) 16 | -------------------------------------------------------------------------------- /inst/CITATION: -------------------------------------------------------------------------------- 1 | utils::bibentry( 2 | header = "Please cite 'future' and the future framework using the following references:", 3 | 4 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 5 | # BibTeX entry: 6 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 7 | bibtype = "article", 8 | key = "RJ-2021-048", 9 | author = "Henrik Bengtsson", 10 | title = "A Unifying Framework for Parallel and Distributed Processing in R using Futures", 11 | year = "2021", 12 | journal = "The R Journal", 13 | doi = "10.32614/RJ-2021-048", 14 | url = "https://doi.org/10.32614/RJ-2021-048", 15 | pages = "208--227", 16 | volume = "13", 17 | number = "2", 18 | 19 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 20 | # Plain-text citation: 21 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 22 | textVersion = paste0( 23 | "Henrik Bengtsson, ", 24 | "A Unifying Framework for Parallel and Distributed Processing in R using Futures, ", 25 | "The R Journal (2021) 13:2, pages 208-227, ", 26 | "doi:10.32614/RJ-2021-048" 27 | ) 28 | ) 29 | -------------------------------------------------------------------------------- /inst/WORDLIST: -------------------------------------------------------------------------------- 1 | alloted 2 | AppVeyor 3 | batchtools 4 | Batchtools 5 | BatchtoolsFuture 6 | CMD 7 | FASTQ 8 | FutureError 9 | GiB 10 | globals 11 | Globals 12 | HPC 13 | lapply 14 | LSF 15 | macOS 16 | multiprocess 17 | OpenLava 18 | pre 19 | Pre 20 | se 21 | SGE 22 | Slurm 23 | th 24 | uniprocess 25 | 26 | ae 27 | availableCores 28 | BAM 29 | bams 30 | BatchJobs 31 | BatchtoolsFutureError 32 | Bengtsson 33 | callr 34 | chr 35 | conf 36 | cran 37 | dir 38 | doJobCollection 39 | evaluator's 40 | fastq 41 | findConfFile 42 | fq 43 | fqs 44 | gb 45 | github 46 | halfCores 47 | htseq 48 | https 49 | io 50 | listenv 51 | logfile 52 | loggedOutput 53 | lsf 54 | mandelbrot 55 | mc 56 | mllg 57 | nbrOfWorkers 58 | nnn 59 | oe 60 | openlava 61 | ppn 62 | readLog 63 | Rscript 64 | rsp 65 | sge 66 | slurm 67 | sprintf 68 | ss 69 | Sys 70 | tmpl 71 | unlist 72 | uri 73 | VignetteAuthor 74 | VignetteEngine 75 | VignetteIndexEntry 76 | VignetteKeyword 77 | VignetteTangle 78 | vmem 79 | wikipedia 80 | multisession 81 | -------------------------------------------------------------------------------- /inst/templates-for-R_CMD_check/batchtools.conf.R: -------------------------------------------------------------------------------- 1 | cluster.functions <- batchtools::makeClusterFunctionsInteractive(external = TRUE) 2 | -------------------------------------------------------------------------------- /inst/templates-for-R_CMD_check/batchtools.sge.tmpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ## Job name: 4 | #$ -N fbt-<%= job.hash %> 5 | 6 | ## Merge standard error and output: 7 | #$ -j y 8 | 9 | ## Direct streams to logfile: 10 | #$ -o <%= log.file %> 11 | 12 | ## Email on abort (a) and termination (e), but not when starting (b) 13 | #PBS -m ae 14 | 15 | ## Tell the queue system to use the current directory 16 | ## as the working directory 17 | ## -cwd 18 | 19 | ## Mirror environment variables 20 | #$ -V 21 | 22 | ## Resources needed: 23 | <% if (exists("resources", mode = "list") && length(resources) > 0) { 24 | if (isTRUE(getOption("future.debug"))) { 25 | R.utils::mcat("resources:") 26 | R.utils::mstr(resources) 27 | } 28 | cat(sprintf("#$ %s\n", resources[["custom"]])) 29 | } %> 30 | 31 | ## UCSF HPC Pilot specific 32 | #$ -S /bin/bash # Required 33 | #$ -l mem_free=1G # Memory usage, required. Note that this is per slot 34 | #$ -R yes # SGE host reservation, highly recommended 35 | 36 | ## SPECIAL: For R CMD check package testing on HPC environments, which 37 | ## typically uses a temporary working directory that is local, we force 38 | ## it to use HPC-wide working directory 39 | #$ -wd ~/tmp 40 | 41 | ## SPECIAL: Since we change working directory, the 'startup.Rs' file used 42 | ## by R CMD check is no longer found 43 | export R_TESTS= 44 | 45 | # For troubleshooting if there are errors 46 | date 47 | hostname 48 | which Rscript 49 | Rscript --version 50 | Rscript -e ".libPaths()" 51 | 52 | echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..." 53 | Rscript -e 'batchtools::doJobCollection("<%= uri %>")' 54 | echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done" 55 | 56 | ## For troubleshooting if there are errors 57 | ## https://github.com/UCSF-HPC/pilot-testing/issues/1 58 | qstat -j $JOB_ID 59 | -------------------------------------------------------------------------------- /inst/templates-for-R_CMD_check/batchtools.torque.tmpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ## Job name: 4 | #PBS -N fbt-<%= job.hash %> 5 | 6 | ## Merge standard error and output: 7 | #PBS -j oe 8 | 9 | ## Direct streams to logfile: 10 | #PBS -o <%= log.file %> 11 | 12 | ## Email on abort (a) and termination (e), but not when starting (b) 13 | #PBS -m ae 14 | 15 | ## Mirror environment variables 16 | #PBS -V 17 | 18 | ## Resources needed: 19 | <% if (exists("resources", mode = "list") && length(resources) > 0) { 20 | if (isTRUE(getOption("future.debug"))) { 21 | R.utils::mcat("resources:") 22 | R.utils::mstr(resources) 23 | } 24 | opts <- unlist(resources, use.names = TRUE) 25 | opts <- sprintf("%s=%s", names(opts), opts) 26 | opts <- paste(opts, collapse = ",") %> 27 | #PBS -l <%= opts %> 28 | <% } %> 29 | 30 | ## SPECIAL: For R CMD check package testing on HPC environments, which 31 | ## typically uses a temporary working directory that is local, we force 32 | ## it to use HPC-wide working directory 33 | #PBS -d <%= normalizePath(file.path("~", "tmp")) %> 34 | 35 | ## SPECIAL: Since we change working directory, the 'startup.Rs' file used 36 | ## by R CMD check is no longer found 37 | export R_TESTS= 38 | 39 | # For troubleshooting if there are errors 40 | date 41 | hostname 42 | which Rscript 43 | Rscript --version 44 | Rscript -e ".libPaths()" 45 | 46 | echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..." 47 | Rscript -e 'batchtools::doJobCollection("<%= uri %>")' 48 | echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done" 49 | 50 | -------------------------------------------------------------------------------- /inst/templates-for-R_CMD_check/setup.R: -------------------------------------------------------------------------------- 1 | ## When running R CMD check on HPC environment with a job scheduler, 2 | ## the batchtools registry files must be written to a location on the 3 | ## file system that is accessible from all compute nodes. This is 4 | ## typically not the case with the default tempdir()/TMPDIR, which 5 | ## often is local and unique to each machine 6 | 7 | tmpdir <- tempfile(pattern = "R_CMD_check_", 8 | tmpdir = file.path("~", "tmp", ".future.batchtools")) 9 | if (!utils::file_test("-d", tmpdir)) { 10 | dir.create(tmpdir, recursive = TRUE) 11 | if (!utils::file_test("-d", tmpdir)) { 12 | stop("R_BATCHTOOLS_SEARCH_PATH/setup.R: Failed to create folder: ", 13 | sQuote(tmpdir)) 14 | } 15 | } 16 | 17 | ## Force the .future/ folders to be in this folder 18 | Sys.setenv("R_FUTURE_CACHE_PATH" = file.path(tmpdir, ".future")) 19 | 20 | ## Make batchtools_ backends use this as their working directory 21 | registry <- list(work.dir = tmpdir) 22 | batchtools_sge <- future::tweak(future.batchtools::batchtools_sge, registry = registry) 23 | -------------------------------------------------------------------------------- /inst/templates/bash.tmpl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ###################################################################### 3 | # A batchtools launch script template for Bash 4 | # 5 | # Author: Henrik Bengtsson 6 | ###################################################################### 7 | 8 | ## Launch R and evaluated the batchtools R job 9 | Rscript -e 'batchtools::doJobCollection("<%= uri %>", output = "<%= log.file %>")' 10 | -------------------------------------------------------------------------------- /inst/templates/sge.tmpl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ###################################################################### 3 | # A batchtools launch script template for SGE 4 | # 5 | # Author: Henrik Bengtsson 6 | ###################################################################### 7 | ## Shell: 8 | #$ -S /bin/bash # Run this as a bash script (required) 9 | 10 | ## Job name: 11 | #$ -N <%= if (exists("job.name", mode = "character")) job.name else job.hash %> 12 | 13 | ## Merge standard error and output: 14 | #$ -j y 15 | 16 | ## Direct streams to logfile: 17 | #$ -o <%= log.file %> 18 | 19 | ## Tell the queue system to use the current directory 20 | ## as the working directory 21 | #$ -cwd 22 | 23 | ## Use environment variables 24 | #$ -V 25 | 26 | ## Resources needed: 27 | <% if (exists("resources", mode = "list") && length(resources) > 0) { 28 | ## As-is resource specifications, e.g. 29 | ## sge_options <- c("-pe smp 2", "-R yes") 30 | ## plan(batchtools_sge, resources = list(asis = sge_options)) 31 | if ("asis" %in% names(resources)) { 32 | cat(sprintf("#$ %s\n", resources[["asis"]])) 33 | resources <- resources[names(resources) != "asis"] 34 | } 35 | 36 | ## Remaining resources are assumed to be of type '-l', e.g. 37 | ## plan(batchtools_sge, resources = list(mem_free = "1G", h_rt="00:20:00")) 38 | opts <- unlist(resources, use.names = TRUE) 39 | opts <- sprintf("-l %s=%s", names(opts), opts) 40 | cat(sprintf("#$ %s\n", opts)) 41 | } %> 42 | 43 | # For troubleshooting, in case there are errors 44 | date 45 | hostname 46 | which Rscript 47 | Rscript --version 48 | Rscript -e ".libPaths()" 49 | 50 | echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..." 51 | Rscript -e 'batchtools::doJobCollection("<%= uri %>")' 52 | res=$? 53 | echo " - exit code: ${res}" 54 | echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done" 55 | 56 | ## For troubleshooting if there are errors 57 | qstat -j $JOB_ID 58 | 59 | ## Relay the exit code from Rscript 60 | exit "$res" 61 | -------------------------------------------------------------------------------- /inst/templates/slurm.tmpl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ###################################################################### 3 | # A batchtools launch script template for Slurm 4 | # 5 | # Author: Henrik Bengtsson 6 | ###################################################################### 7 | 8 | #SBATCH --job-name=<%= job.name %> 9 | #SBATCH --output=<%= log.file %> 10 | #SBATCH --nodes=1 11 | #SBATCH --time=00:05:00 12 | 13 | ## Resources needed: 14 | <% if (length(resources) > 0) { 15 | opts <- unlist(resources, use.names = TRUE) 16 | opts <- sprintf("--%s=%s", names(opts), opts) 17 | opts <- paste(opts, collapse = " ") %> 18 | #SBATCH <%= opts %> 19 | <% } %> 20 | 21 | ## Launch R and evaluated the batchtools R job 22 | Rscript -e 'batchtools::doJobCollection("<%= uri %>")' 23 | -------------------------------------------------------------------------------- /inst/templates/torque.tmpl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ###################################################################### 3 | # A batchtools launch script template for Torque/PBS 4 | # 5 | # Author: Henrik Bengtsson 6 | ###################################################################### 7 | 8 | ## Job name: 9 | #PBS -N <%= if (exists("job.name", mode = "character")) job.name else job.hash %> 10 | 11 | ## Direct streams to logfile: 12 | #PBS -o <%= log.file %> 13 | 14 | ## Merge standard error and output: 15 | #PBS -j oe 16 | 17 | ## Email on abort (a) and termination (e), but not when starting (b) 18 | #PBS -m ae 19 | 20 | ## Resources needed: 21 | <% if (length(resources) > 0) { 22 | opts <- unlist(resources, use.names = TRUE) 23 | opts <- sprintf("%s=%s", names(opts), opts) 24 | opts <- paste(opts, collapse = ",") %> 25 | #PBS -l <%= opts %> 26 | <% } %> 27 | 28 | ## Launch R and evaluated the batchtools R job 29 | Rscript -e 'batchtools::doJobCollection("<%= uri %>")' 30 | -------------------------------------------------------------------------------- /man/BatchtoolsFuture.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/BatchtoolsCustomFuture-class.R, 3 | % R/BatchtoolsFuture-class.R, R/BatchtoolsMultiprocessFuture-class.R, 4 | % R/BatchtoolsSSHFuture-class.R, R/BatchtoolsUniprocessFuture-class.R 5 | \name{BatchtoolsCustomFuture} 6 | \alias{BatchtoolsCustomFuture} 7 | \alias{BatchtoolsBashFuture} 8 | \alias{BatchtoolsFuture} 9 | \alias{BatchtoolsMultiprocessFuture} 10 | \alias{BatchtoolsMulticoreFuture} 11 | \alias{BatchtoolsTemplateFuture} 12 | \alias{BatchtoolsLsfFuture} 13 | \alias{BatchtoolsOpenLavaFuture} 14 | \alias{BatchtoolsSGEFuture} 15 | \alias{BatchtoolsSlurmFuture} 16 | \alias{BatchtoolsTorqueFuture} 17 | \alias{BatchtoolsSSHFuture} 18 | \alias{BatchtoolsUniprocessFuture} 19 | \alias{BatchtoolsLocalFuture} 20 | \alias{BatchtoolsInteractiveFuture} 21 | \title{A batchtools future is a future whose value will be resolved via batchtools} 22 | \usage{ 23 | BatchtoolsCustomFuture( 24 | expr = NULL, 25 | substitute = TRUE, 26 | envir = parent.frame(), 27 | ... 28 | ) 29 | 30 | BatchtoolsBashFuture( 31 | expr = NULL, 32 | substitute = TRUE, 33 | envir = parent.frame(), 34 | ... 35 | ) 36 | 37 | BatchtoolsFuture( 38 | expr = NULL, 39 | envir = parent.frame(), 40 | substitute = TRUE, 41 | globals = TRUE, 42 | packages = NULL, 43 | label = NULL, 44 | resources = list(), 45 | workers = NULL, 46 | finalize = getOption("future.finalize", TRUE), 47 | conf.file = findConfFile(), 48 | cluster.functions = NULL, 49 | registry = list(), 50 | ... 51 | ) 52 | 53 | BatchtoolsMultiprocessFuture( 54 | expr = NULL, 55 | substitute = TRUE, 56 | envir = parent.frame(), 57 | ... 58 | ) 59 | 60 | BatchtoolsMulticoreFuture( 61 | expr = NULL, 62 | substitute = TRUE, 63 | envir = parent.frame(), 64 | ... 65 | ) 66 | 67 | BatchtoolsTemplateFuture( 68 | expr = NULL, 69 | substitute = TRUE, 70 | envir = parent.frame(), 71 | ... 72 | ) 73 | 74 | BatchtoolsLsfFuture( 75 | expr = NULL, 76 | substitute = TRUE, 77 | envir = parent.frame(), 78 | ... 79 | ) 80 | 81 | BatchtoolsOpenLavaFuture( 82 | expr = NULL, 83 | substitute = TRUE, 84 | envir = parent.frame(), 85 | ... 86 | ) 87 | 88 | BatchtoolsSGEFuture( 89 | expr = NULL, 90 | substitute = TRUE, 91 | envir = parent.frame(), 92 | ... 93 | ) 94 | 95 | BatchtoolsSlurmFuture( 96 | expr = NULL, 97 | substitute = TRUE, 98 | envir = parent.frame(), 99 | ... 100 | ) 101 | 102 | BatchtoolsTorqueFuture( 103 | expr = NULL, 104 | substitute = TRUE, 105 | envir = parent.frame(), 106 | ... 107 | ) 108 | 109 | BatchtoolsSSHFuture( 110 | expr = NULL, 111 | substitute = TRUE, 112 | envir = parent.frame(), 113 | workers = availableCores(), 114 | ... 115 | ) 116 | 117 | BatchtoolsUniprocessFuture( 118 | expr = NULL, 119 | substitute = TRUE, 120 | envir = parent.frame(), 121 | ... 122 | ) 123 | 124 | BatchtoolsLocalFuture( 125 | expr = NULL, 126 | substitute = TRUE, 127 | envir = parent.frame(), 128 | ... 129 | ) 130 | 131 | BatchtoolsInteractiveFuture( 132 | expr = NULL, 133 | substitute = TRUE, 134 | envir = parent.frame(), 135 | ... 136 | ) 137 | } 138 | \arguments{ 139 | \item{expr}{The R expression to be evaluated} 140 | 141 | \item{substitute}{Controls whether \code{expr} should be 142 | \code{substitute()}:d or not.} 143 | 144 | \item{envir}{The environment in which global environment 145 | should be located.} 146 | 147 | \item{globals}{(optional) a logical, a character vector, a named list, or a 148 | \link[globals:Globals]{Globals} object. If TRUE, globals are identified by code 149 | inspection based on \code{expr} and \code{tweak} searching from environment 150 | \code{envir}. If FALSE, no globals are used. If a character vector, then 151 | globals are identified by lookup based their names \code{globals} searching 152 | from environment \code{envir}. If a named list or a Globals object, the 153 | globals are used as is.} 154 | 155 | \item{label}{(optional) Label of the future (where applicable, becomes the 156 | job name for most job schedulers).} 157 | 158 | \item{resources}{(optional) A named list passed to the \pkg{batchtools} 159 | template (available as variable \code{resources}). See Section 'Resources' 160 | in \code{\link[batchtools:submitJobs]{batchtools::submitJobs()}} more details.} 161 | 162 | \item{workers}{(optional) The maximum number of workers the batchtools 163 | backend may use at any time. Interactive and "local" backends can only 164 | process one future at the time (\code{workers = 1L}), whereas HPC backends, 165 | where futures are resolved via separate jobs on a scheduler, can have 166 | multiple workers. In the latter, the default is \code{workers = NULL}, which 167 | will resolve to 168 | \code{getOption("\link{future.batchtools.workers}")}. 169 | If neither are specified, then the default is \code{100}.} 170 | 171 | \item{finalize}{If TRUE, any underlying registries are 172 | deleted when this object is garbage collected, otherwise not.} 173 | 174 | \item{conf.file}{(optional) A batchtools configuration file.} 175 | 176 | \item{cluster.functions}{(optional) A batchtools 177 | \link[batchtools:makeClusterFunctions]{ClusterFunctions} object.} 178 | 179 | \item{registry}{(optional) A named list of settings to control the setup 180 | of the batchtools registry.} 181 | 182 | \item{\ldots}{Additional arguments passed to \code{\link[future:Future-class]{future::Future()}}.} 183 | } 184 | \value{ 185 | A BatchtoolsFuture object 186 | } 187 | \description{ 188 | A batchtools future is a future whose value will be resolved via batchtools 189 | } 190 | \keyword{internal} 191 | -------------------------------------------------------------------------------- /man/BatchtoolsFutureError.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/BatchtoolsFutureError.R 3 | \name{BatchtoolsFutureError} 4 | \alias{BatchtoolsFutureError} 5 | \title{FutureError class for errors related to BatchtoolsFuture:s} 6 | \usage{ 7 | BatchtoolsFutureError(...) 8 | } 9 | \arguments{ 10 | \item{\ldots}{Arguments passed to \link[future:FutureCondition]{FutureError}.} 11 | } 12 | \description{ 13 | FutureError class for errors related to BatchtoolsFuture:s 14 | } 15 | \keyword{internal} 16 | -------------------------------------------------------------------------------- /man/batchtools_custom.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/batchtools_custom.R 3 | \name{batchtools_custom} 4 | \alias{batchtools_custom} 5 | \title{Batchtools futures for custom batchtools configuration} 6 | \usage{ 7 | batchtools_custom( 8 | expr, 9 | envir = parent.frame(), 10 | substitute = TRUE, 11 | globals = TRUE, 12 | label = NULL, 13 | resources = list(), 14 | workers = NULL, 15 | conf.file = findConfFile(), 16 | cluster.functions = NULL, 17 | registry = list(), 18 | ... 19 | ) 20 | } 21 | \arguments{ 22 | \item{expr}{The R expression to be evaluated} 23 | 24 | \item{envir}{The environment in which global environment 25 | should be located.} 26 | 27 | \item{substitute}{Controls whether \code{expr} should be 28 | \code{substitute()}:d or not.} 29 | 30 | \item{globals}{(optional) a logical, a character vector, a named list, or a 31 | \link[globals:Globals]{Globals} object. If TRUE, globals are identified by code 32 | inspection based on \code{expr} and \code{tweak} searching from environment 33 | \code{envir}. If FALSE, no globals are used. If a character vector, then 34 | globals are identified by lookup based their names \code{globals} searching 35 | from environment \code{envir}. If a named list or a Globals object, the 36 | globals are used as is.} 37 | 38 | \item{label}{(optional) Label of the future (where applicable, becomes the 39 | job name for most job schedulers).} 40 | 41 | \item{resources}{(optional) A named list passed to the \pkg{batchtools} 42 | template (available as variable \code{resources}). See Section 'Resources' 43 | in \code{\link[batchtools:submitJobs]{batchtools::submitJobs()}} more details.} 44 | 45 | \item{workers}{(optional) The maximum number of workers the batchtools 46 | backend may use at any time. Interactive and "local" backends can only 47 | process one future at the time (\code{workers = 1L}), whereas HPC backends, 48 | where futures are resolved via separate jobs on a scheduler, can have 49 | multiple workers. In the latter, the default is \code{workers = NULL}, which 50 | will resolve to 51 | \code{getOption("\link{future.batchtools.workers}")}. 52 | If neither are specified, then the default is \code{100}.} 53 | 54 | \item{conf.file}{(character) A batchtools configuration file as for 55 | instance returned by \code{\link[batchtools:findConfFile]{batchtools::findConfFile()}}.} 56 | 57 | \item{cluster.functions}{A 58 | \link[batchtools:makeClusterFunctions]{ClusterFunctions} object.} 59 | 60 | \item{registry}{(optional) A named list of settings to control the setup 61 | of the batchtools registry.} 62 | 63 | \item{\ldots}{Additional arguments passed to \code{\link[=BatchtoolsFuture]{BatchtoolsFuture()}}.} 64 | } 65 | \value{ 66 | An object of class \code{BatchtoolsFuture}. 67 | } 68 | \description{ 69 | Batchtools futures for custom batchtools configuration 70 | } 71 | \examples{ 72 | options(error = function(...) { 73 | print(traceback()) 74 | }) 75 | 76 | cf <- batchtools::makeClusterFunctionsInteractive(external = TRUE) 77 | print(cf) 78 | str(cf) 79 | plan(batchtools_custom, cluster.functions = cf) 80 | print(plan()) 81 | print(nbrOfWorkers()) 82 | 83 | ## Create explicit future 84 | f <- future({ 85 | cat("PID:", Sys.getpid(), "\n") 86 | 42L 87 | }) 88 | print(f) 89 | v <- value(f) 90 | print(v) 91 | 92 | options(error = NULL) 93 | 94 | 95 | ## Create explicit future 96 | f <- future({ 97 | cat("PID:", Sys.getpid(), "\n") 98 | 42L 99 | }) 100 | print(f) 101 | v <- value(f) 102 | print(v) 103 | 104 | 105 | 106 | ## Create explicit future 107 | f <- future({ 108 | cat("PID:", Sys.getpid(), "\n") 109 | 42L 110 | }) 111 | v <- value(f) 112 | print(v) 113 | } 114 | -------------------------------------------------------------------------------- /man/batchtools_local.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/batchtools_local.R 3 | \name{batchtools_local} 4 | \alias{batchtools_local} 5 | \alias{batchtools_interactive} 6 | \alias{batchtools_bash} 7 | \title{batchtools local and interactive futures} 8 | \usage{ 9 | batchtools_local(..., envir = parent.frame()) 10 | } 11 | \arguments{ 12 | \item{envir}{The environment in which global environment 13 | should be located.} 14 | 15 | \item{\ldots}{Additional arguments passed to \code{\link[=BatchtoolsUniprocessFuture]{BatchtoolsUniprocessFuture()}}.} 16 | } 17 | \value{ 18 | An object of class \code{BatchtoolsUniprocessFuture}. 19 | } 20 | \description{ 21 | A batchtools local future is an synchronous uniprocess future that 22 | will be evaluated in a background R session. 23 | A batchtools interactive future is an synchronous uniprocess future 24 | that will be evaluated in the current R session (and variables will 25 | be assigned to the calling environment rather than to a local one). 26 | Both types of futures will block until the futures are resolved. 27 | } 28 | \details{ 29 | batchtools local futures rely on the batchtools backend set up by 30 | \code{\link[batchtools:makeClusterFunctionsInteractive]{batchtools::makeClusterFunctionsInteractive(external = TRUE)}} 31 | and batchtools interactive futures on the one set up by 32 | \code{\link[batchtools:makeClusterFunctionsInteractive]{batchtools::makeClusterFunctionsInteractive()}}. 33 | These are supported by all operating systems. 34 | 35 | An alternative to batchtools local futures is to use 36 | \link[future:cluster]{cluster} futures of the \pkg{future} 37 | package with a single local background session, i.e. 38 | \code{plan(cluster, workers = "localhost")}. 39 | 40 | An alternative to batchtools interactive futures is to use 41 | \code{plan(sequential, split = TRUE)} futures of the \pkg{future} package. 42 | } 43 | \examples{ 44 | ## Use local batchtools futures 45 | plan(batchtools_local) 46 | 47 | ## A global variable 48 | a <- 1 49 | 50 | ## Create explicit future 51 | f <- future({ 52 | b <- 3 53 | c <- 2 54 | a * b * c 55 | }) 56 | v <- value(f) 57 | print(v) 58 | 59 | 60 | ## Create implicit future 61 | v \%<-\% { 62 | b <- 3 63 | c <- 2 64 | a * b * c 65 | } 66 | print(v) 67 | } 68 | -------------------------------------------------------------------------------- /man/batchtools_multicore.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/batchtools_multicore.R 3 | \name{batchtools_multicore} 4 | \alias{batchtools_multicore} 5 | \title{batchtools multicore futures} 6 | \usage{ 7 | batchtools_multicore( 8 | expr, 9 | envir = parent.frame(), 10 | substitute = TRUE, 11 | globals = TRUE, 12 | label = NULL, 13 | workers = availableCores(constraints = "multicore"), 14 | registry = list(), 15 | ... 16 | ) 17 | } 18 | \arguments{ 19 | \item{expr}{The R expression to be evaluated} 20 | 21 | \item{envir}{The environment in which global environment 22 | should be located.} 23 | 24 | \item{substitute}{Controls whether \code{expr} should be 25 | \code{substitute()}:d or not.} 26 | 27 | \item{globals}{(optional) a logical, a character vector, a named list, or a 28 | \link[globals:Globals]{Globals} object. If TRUE, globals are identified by code 29 | inspection based on \code{expr} and \code{tweak} searching from environment 30 | \code{envir}. If FALSE, no globals are used. If a character vector, then 31 | globals are identified by lookup based their names \code{globals} searching 32 | from environment \code{envir}. If a named list or a Globals object, the 33 | globals are used as is.} 34 | 35 | \item{label}{(optional) Label of the future (where applicable, becomes the 36 | job name for most job schedulers).} 37 | 38 | \item{workers}{The number of multicore processes to be 39 | available for concurrent batchtools multicore futures.} 40 | 41 | \item{registry}{(optional) A named list of settings to control the setup 42 | of the batchtools registry.} 43 | 44 | \item{\ldots}{Additional arguments passed 45 | to \code{\link[=BatchtoolsFuture]{BatchtoolsFuture()}}.} 46 | } 47 | \value{ 48 | An object of class \code{BatchtoolsMulticoreFuture}. 49 | } 50 | \description{ 51 | A batchtools multicore future is an asynchronous multiprocess 52 | future that will be evaluated in a background R session.\cr 53 | \cr 54 | \emph{We highly recommend using \link[future:multisession]{future::multisession} 55 | (sic!) futures of the \pkg{future} package instead of 56 | multicore batchtools futures.} 57 | } 58 | \details{ 59 | batchtools multicore futures rely on the batchtools backend set 60 | up by \code{\link[batchtools:makeClusterFunctionsMulticore]{batchtools::makeClusterFunctionsMulticore()}}. 61 | The batchtools multicore backend only works on operating systems 62 | supporting the \code{ps} command-line tool, e.g. Linux and macOS. 63 | } 64 | \keyword{internal} 65 | -------------------------------------------------------------------------------- /man/batchtools_ssh.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/batchtools_ssh.R 3 | \name{batchtools_ssh} 4 | \alias{batchtools_ssh} 5 | \title{batchtools SSH futures} 6 | \usage{ 7 | batchtools_ssh( 8 | expr, 9 | envir = parent.frame(), 10 | substitute = TRUE, 11 | globals = TRUE, 12 | label = NULL, 13 | workers = availableWorkers(), 14 | registry = list(), 15 | ... 16 | ) 17 | } 18 | \arguments{ 19 | \item{expr}{The R expression to be evaluated} 20 | 21 | \item{envir}{The environment in which global environment 22 | should be located.} 23 | 24 | \item{substitute}{Controls whether \code{expr} should be 25 | \code{substitute()}:d or not.} 26 | 27 | \item{globals}{(optional) a logical, a character vector, a named list, or a 28 | \link[globals:Globals]{Globals} object. If TRUE, globals are identified by code 29 | inspection based on \code{expr} and \code{tweak} searching from environment 30 | \code{envir}. If FALSE, no globals are used. If a character vector, then 31 | globals are identified by lookup based their names \code{globals} searching 32 | from environment \code{envir}. If a named list or a Globals object, the 33 | globals are used as is.} 34 | 35 | \item{label}{(optional) Label of the future (where applicable, becomes the 36 | job name for most job schedulers).} 37 | 38 | \item{workers}{The number of SSH processes to be 39 | available for concurrent batchtools SSH futures.} 40 | 41 | \item{registry}{(optional) A named list of settings to control the setup 42 | of the batchtools registry.} 43 | 44 | \item{\ldots}{Additional arguments passed 45 | to \code{\link[=BatchtoolsFuture]{BatchtoolsFuture()}}.} 46 | } 47 | \value{ 48 | An object of class \code{BatchtoolsMulticoreFuture}. 49 | } 50 | \description{ 51 | A batchtools SSH future is an asynchronous multiprocess 52 | future that will be evaluated in a background R session.\cr 53 | \cr 54 | \emph{We highly recommend using \link[future:multisession]{future::multisession} 55 | (sic!) futures of the \pkg{future} package instead of 56 | SSH batchtools futures.} 57 | } 58 | \details{ 59 | batchtools SSH futures rely on the batchtools backend set 60 | up by \code{\link[batchtools:makeClusterFunctionsSSH]{batchtools::makeClusterFunctionsSSH()}}. 61 | The batchtools SSH backend only works on operating systems 62 | supporting the \code{ssh} and \code{ps} command-line tool, e.g. Linux and macOS. 63 | } 64 | \keyword{internal} 65 | -------------------------------------------------------------------------------- /man/batchtools_template.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/batchtools_template.R 3 | \name{batchtools_template} 4 | \alias{batchtools_template} 5 | \alias{batchtools_lsf} 6 | \alias{batchtools_openlava} 7 | \alias{batchtools_sge} 8 | \alias{batchtools_slurm} 9 | \alias{batchtools_torque} 10 | \title{Batchtools futures for LSF, OpenLava, SGE, Slurm, TORQUE etc.} 11 | \usage{ 12 | batchtools_lsf( 13 | expr, 14 | envir = parent.frame(), 15 | substitute = TRUE, 16 | globals = TRUE, 17 | label = NULL, 18 | template = NULL, 19 | resources = list(), 20 | workers = NULL, 21 | registry = list(), 22 | ... 23 | ) 24 | 25 | batchtools_openlava( 26 | expr, 27 | envir = parent.frame(), 28 | substitute = TRUE, 29 | globals = TRUE, 30 | label = NULL, 31 | template = NULL, 32 | resources = list(), 33 | workers = NULL, 34 | registry = list(), 35 | ... 36 | ) 37 | 38 | batchtools_sge( 39 | expr, 40 | envir = parent.frame(), 41 | substitute = TRUE, 42 | globals = TRUE, 43 | label = NULL, 44 | template = NULL, 45 | resources = list(), 46 | workers = NULL, 47 | registry = list(), 48 | ... 49 | ) 50 | 51 | batchtools_slurm( 52 | expr, 53 | envir = parent.frame(), 54 | substitute = TRUE, 55 | globals = TRUE, 56 | label = NULL, 57 | template = NULL, 58 | resources = list(), 59 | workers = NULL, 60 | registry = list(), 61 | ... 62 | ) 63 | 64 | batchtools_torque( 65 | expr, 66 | envir = parent.frame(), 67 | substitute = TRUE, 68 | globals = TRUE, 69 | label = NULL, 70 | template = NULL, 71 | resources = list(), 72 | workers = NULL, 73 | registry = list(), 74 | ... 75 | ) 76 | } 77 | \arguments{ 78 | \item{expr}{The R expression to be evaluated} 79 | 80 | \item{envir}{The environment in which global environment 81 | should be located.} 82 | 83 | \item{substitute}{Controls whether \code{expr} should be 84 | \code{substitute()}:d or not.} 85 | 86 | \item{globals}{(optional) a logical, a character vector, a named list, or a 87 | \link[globals:Globals]{Globals} object. If TRUE, globals are identified by code 88 | inspection based on \code{expr} and \code{tweak} searching from environment 89 | \code{envir}. If FALSE, no globals are used. If a character vector, then 90 | globals are identified by lookup based their names \code{globals} searching 91 | from environment \code{envir}. If a named list or a Globals object, the 92 | globals are used as is.} 93 | 94 | \item{label}{(optional) Label of the future (where applicable, becomes the 95 | job name for most job schedulers).} 96 | 97 | \item{template}{(optional) A batchtools template file or a template string 98 | (in \pkg{brew} format). If not specified, it is left to the 99 | \pkg{batchtools} package to locate such file using its search rules.} 100 | 101 | \item{resources}{(optional) A named list passed to the \pkg{batchtools} 102 | template (available as variable \code{resources}). See Section 'Resources' 103 | in \code{\link[batchtools:submitJobs]{batchtools::submitJobs()}} more details.} 104 | 105 | \item{workers}{(optional) The maximum number of workers the batchtools 106 | backend may use at any time. Interactive and "local" backends can only 107 | process one future at the time (\code{workers = 1L}), whereas HPC backends, 108 | where futures are resolved via separate jobs on a scheduler, can have 109 | multiple workers. In the latter, the default is \code{workers = NULL}, which 110 | will resolve to 111 | \code{getOption("\link{future.batchtools.workers}")}. 112 | If neither are specified, then the default is \code{100}.} 113 | 114 | \item{registry}{(optional) A named list of settings to control the setup 115 | of the batchtools registry.} 116 | 117 | \item{\ldots}{Additional arguments passed to \code{\link[=BatchtoolsFuture]{BatchtoolsFuture()}}.} 118 | } 119 | \value{ 120 | An object of class \code{BatchtoolsFuture}. 121 | } 122 | \description{ 123 | Batchtools futures for LSF, OpenLava, SGE, Slurm, TORQUE etc. are 124 | asynchronous multiprocess futures that will be evaluated on a compute 125 | cluster via a job scheduler. 126 | } 127 | \details{ 128 | These type of batchtools futures rely on batchtools backends set 129 | up using the following \pkg{batchtools} functions: 130 | \itemize{ 131 | \item \code{\link[batchtools:makeClusterFunctionsLSF]{batchtools::makeClusterFunctionsLSF()}} for 132 | \href{https://en.wikipedia.org/wiki/Platform_LSF}{Load Sharing Facility (LSF)} 133 | \item \code{\link[batchtools:makeClusterFunctionsOpenLava]{batchtools::makeClusterFunctionsOpenLava()}} for 134 | \href{https://en.wikipedia.org/wiki/OpenLava}{OpenLava} 135 | \item \code{\link[batchtools:makeClusterFunctionsSGE]{batchtools::makeClusterFunctionsSGE()}} for 136 | \href{https://en.wikipedia.org/wiki/Oracle_Grid_Engine}{Sun/Oracle Grid Engine (SGE)} 137 | \item \code{\link[batchtools:makeClusterFunctionsSlurm]{batchtools::makeClusterFunctionsSlurm()}} for 138 | \href{https://en.wikipedia.org/wiki/Slurm_Workload_Manager}{Slurm} 139 | \item \code{\link[batchtools:makeClusterFunctionsTORQUE]{batchtools::makeClusterFunctionsTORQUE()}} for 140 | \href{https://en.wikipedia.org/wiki/TORQUE}{TORQUE} / PBS 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /man/delete.BatchtoolsFuture.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/BatchtoolsFuture-class.R 3 | \name{delete.BatchtoolsFuture} 4 | \alias{delete.BatchtoolsFuture} 5 | \title{Removes a batchtools future} 6 | \usage{ 7 | \method{delete}{BatchtoolsFuture}( 8 | future, 9 | onRunning = c("warning", "error", "skip"), 10 | onFailure = c("error", "warning", "ignore"), 11 | onMissing = c("ignore", "warning", "error"), 12 | times = 10L, 13 | delta = getOption("future.wait.interval", 1), 14 | alpha = getOption("future.wait.alpha", 1.01), 15 | ... 16 | ) 17 | } 18 | \arguments{ 19 | \item{future}{The future.} 20 | 21 | \item{onRunning}{Action if future is running or appears to run.} 22 | 23 | \item{onFailure}{Action if failing to delete future.} 24 | 25 | \item{onMissing}{Action if future does not exist.} 26 | 27 | \item{times}{The number of tries before giving up.} 28 | 29 | \item{delta}{The delay interval (in seconds) between retries.} 30 | 31 | \item{alpha}{A multiplicative penalty increasing the delay 32 | for each failed try.} 33 | 34 | \item{\ldots}{Not used.} 35 | } 36 | \value{ 37 | (invisibly) TRUE if deleted and FALSE otherwise. 38 | } 39 | \description{ 40 | Removes a batchtools future 41 | } 42 | \keyword{internal} 43 | -------------------------------------------------------------------------------- /man/future.batchtools.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/future.batchtools-package.R 3 | \docType{package} 4 | \name{future.batchtools} 5 | \alias{future.batchtools} 6 | \alias{future.batchtools-package} 7 | \title{future.batchtools: A Future for batchtools} 8 | \description{ 9 | The \pkg{future.batchtools} package implements the Future API 10 | on top of \pkg{batchtools} such that futures can be resolved 11 | on for instance high-performance compute (HPC) clusters via 12 | job schedulers. 13 | The Future API is defined by the \pkg{future} package. 14 | } 15 | \details{ 16 | To use batchtools futures, load \pkg{future.batchtools}, and 17 | select the type of future you wish to use via 18 | \code{\link[future:plan]{future::plan()}}. 19 | } 20 | \examples{ 21 | library(future.batchtools) 22 | 23 | ## Use local batchtools futures 24 | plan(batchtools_local) 25 | 26 | ## A global variable 27 | a <- 1 28 | 29 | v \%<-\% { 30 | b <- 3 31 | c <- 2 32 | a * b * c 33 | } 34 | 35 | print(v) 36 | \donttest{ 37 | plan(batchtools_local) 38 | demo("mandelbrot", package = "future", ask = FALSE) 39 | } 40 | 41 | } 42 | \seealso{ 43 | Useful links: 44 | \itemize{ 45 | \item \url{https://future.batchtools.futureverse.org} 46 | \item \url{https://github.com/futureverse/future.batchtools} 47 | \item Report bugs at \url{https://github.com/futureverse/future.batchtools/issues} 48 | } 49 | 50 | } 51 | \author{ 52 | \strong{Maintainer}: Henrik Bengtsson \email{henrikb@braju.com} (\href{https://orcid.org/0000-0002-7579-5165}{ORCID}) [copyright holder] 53 | 54 | } 55 | -------------------------------------------------------------------------------- /man/future.batchtools.options.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/options.R 3 | \name{future.batchtools.options} 4 | \alias{future.batchtools.options} 5 | \alias{future.cache.path} 6 | \alias{future.delete} 7 | \alias{R_FUTURE_CACHE_PATH} 8 | \alias{R_FUTURE_DELETE} 9 | \alias{future.batchtools.expiration.tail} 10 | \alias{future.batchtools.output} 11 | \alias{future.batchtools.workers} 12 | \alias{R_FUTURE_BATCHTOOLS_EXPIRATION_TAIL} 13 | \alias{R_FUTURE_BATCHTOOLS_OUTPUT} 14 | \alias{R_FUTURE_BATCHTOOLS_WORKERS} 15 | \title{Options used for batchtools futures} 16 | \description{ 17 | Below are the \R options and environment variables that are used by the 18 | \pkg{future.batchtools} package. 19 | See \link[future:zzz-future.options]{future::future.options} for additional ones that apply to futures 20 | in general.\cr 21 | \cr 22 | \emph{WARNING: Note that the names and the default values of these options 23 | may change in future versions of the package. Please use with care 24 | until further notice.} 25 | } 26 | \section{Settings for batchtools futures}{ 27 | 28 | \describe{ 29 | \item{\option{future.batchtools.workers}:}{(a positive numeric or \code{+Inf}) 30 | The default number of workers available on HPC schedulers with 31 | job queues. (Default: \code{100})} 32 | 33 | \item{\option{future.batchtools.output}:}{(logical) 34 | If TRUE, \pkg{batchtools} will produce extra output. 35 | If FALSE, such output will be disabled by setting \pkg{batchtools} 36 | options \option{batchtools.verbose} and \option{batchtools.progress} 37 | to FALSE. 38 | (Default: \code{getOption("future.debug", FALSE)})} 39 | 40 | \item{\option{future.batchtools.expiration.tail}:}{(a positive numeric) 41 | When a \pkg{batchtools} job expires, the last few lines will be 42 | relayed by batchtools futures to help troubleshooting. 43 | This option controls how many lines are displayed. 44 | (Default: \code{48L})} 45 | 46 | \item{\option{future.cache.path}:}{ 47 | (character string) 48 | An absolute or relative path specifying the root folder in which 49 | \pkg{batchtools} registry folders are stored. 50 | This folder needs to be accessible from all hosts ("workers"). 51 | Specifically, it must \emph{not} be a folder that is only local to the 52 | machine such as \verb{file.path(tempdir(), ".future"} if an job scheduler 53 | on a HPC environment is used. 54 | (Default: \code{.future} in the current working directory)} 55 | 56 | \item{\option{future.delete}:}{(logical) 57 | Controls whether or not the future's \pkg{batchtools} registry folder 58 | is deleted after the future result has been collected. 59 | If TRUE, it is always deleted. 60 | If FALSE, it is never deleted. 61 | If not set or NULL, the it is deleted, unless running in non-interactive 62 | mode and the batchtools job failed or expired, which helps to 63 | troubleshoot when running in batch mode. 64 | (Default: NULL (not set))} 65 | } 66 | } 67 | 68 | \section{Environment variables that set R options}{ 69 | 70 | All of the above \R \option{future.batchtools.*} options can be set by 71 | corresponding environment variable \env{R_FUTURE_BATCHTOOLS_*} \emph{when 72 | the \pkg{future.batchtools} package is loaded}. This means that those 73 | environment variables must be set before the \pkg{future.batchtools} 74 | package is loaded in order to have an effect. 75 | For example, if \code{R_FUTURE_BATCHTOOLS_WORKERS="200"} is set, then option 76 | \option{future.batchtools.workers} is set to \code{200} (numeric). 77 | } 78 | 79 | \examples{ 80 | # Set an R option: 81 | options(future.cache.path = "/cluster-wide/folder/.future") 82 | 83 | } 84 | -------------------------------------------------------------------------------- /man/grapes-resources-grapes.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/resources_OP.R 3 | \name{\%resources\%} 4 | \alias{\%resources\%} 5 | \title{Temporarily tweaks the resources for the current batchtools strategy} 6 | \usage{ 7 | fassignment \%resources\% tweaks 8 | } 9 | \arguments{ 10 | \item{fassignment}{The future assignment, e.g. 11 | \code{x \%<-\% \{ expr \}}.} 12 | 13 | \item{tweaks}{A named list (or vector) of resource \pkg{batchtools} 14 | parameters (see Section 'Resources' in \code{\link[batchtools:submitJobs]{batchtools::submitJobs()}}) 15 | that should be changed relative to the current strategy.} 16 | } 17 | \description{ 18 | Temporarily tweaks the resources for the current batchtools strategy 19 | } 20 | \keyword{internal} 21 | -------------------------------------------------------------------------------- /man/loggedOutput.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/BatchtoolsFuture-class.R 3 | \name{loggedOutput} 4 | \alias{loggedOutput} 5 | \alias{loggedError} 6 | \alias{loggedError.BatchtoolsFuture} 7 | \title{Logged output of batchtools future} 8 | \usage{ 9 | loggedOutput(...) 10 | 11 | \method{loggedError}{BatchtoolsFuture}(future, ...) 12 | } 13 | \arguments{ 14 | \item{future}{The future.} 15 | 16 | \item{\ldots}{Not used.} 17 | } 18 | \value{ 19 | A character vector or a logical scalar. 20 | } 21 | \description{ 22 | Logged output of batchtools future 23 | } 24 | \keyword{internal} 25 | -------------------------------------------------------------------------------- /man/nbrOfWorkers.batchtools.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/nbrOfWorkers.R 3 | \name{nbrOfWorkers.batchtools} 4 | \alias{nbrOfWorkers.batchtools} 5 | \title{Gets the number of batchtools workers} 6 | \usage{ 7 | \method{nbrOfWorkers}{batchtools}(evaluator) 8 | } 9 | \arguments{ 10 | \item{evaluator}{A future evaluator function. 11 | If NULL (default), the current evaluator as returned 12 | by \code{\link[future:plan]{future::plan()}} is used.} 13 | } 14 | \value{ 15 | A number in \eqn{[1, Inf]}. 16 | } 17 | \description{ 18 | Tries to infer the total number of batchtools workers. This is 19 | done using various ad-hoc procedures based on code inspection 20 | of batchtools itself. 21 | } 22 | \keyword{internal} 23 | -------------------------------------------------------------------------------- /man/print.BatchtoolsFuture.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/BatchtoolsFuture-class.R 3 | \name{print.BatchtoolsFuture} 4 | \alias{print.BatchtoolsFuture} 5 | \title{Prints a batchtools future} 6 | \usage{ 7 | \method{print}{BatchtoolsFuture}(x, ...) 8 | } 9 | \arguments{ 10 | \item{x}{An BatchtoolsFuture object} 11 | 12 | \item{\ldots}{Not used.} 13 | } 14 | \description{ 15 | Prints a batchtools future 16 | } 17 | \keyword{internal} 18 | -------------------------------------------------------------------------------- /pkgdown/_pkgdown.yml: -------------------------------------------------------------------------------- 1 | url: https://future.batchtools.futureverse.org 2 | 3 | home: 4 | links: 5 | - text: Roadmap/Milestones 6 | href: https://github.com/futureverse/future.batchtools/milestones 7 | - text: The Futureverse Project 8 | href: https://www.futureverse.org/ 9 | - text: Futureverse User Forum 10 | href: https://github.com/futureverse/future/discussions 11 | 12 | navbar: 13 | structure: 14 | right: [search, futureverse, pkgs, cran, github, lightswitch] 15 | components: 16 | futureverse: 17 | icon: fas fa-home 18 | href: https://www.futureverse.org/ 19 | pkgs: 20 | text: Packages 21 | menu: 22 | - text: doFuture (map-reduce) 23 | href: https://doFuture.futureverse.org 24 | - text: furrr (map-reduce) 25 | href: https://furrr.futureverse.org 26 | - text: future 27 | href: https://future.futureverse.org 28 | - text: future.apply (map-reduce) 29 | href: https://future.apply.futureverse.org 30 | - text: future.batchtools (backend) 31 | href: https://future.batchtools.futureverse.org 32 | - text: future.callr (backend) 33 | href: https://future.callr.futureverse.org 34 | - text: future.mirai (backend) 35 | href: https://future.mirai.futureverse.org 36 | - text: future.tests 37 | href: https://future.tests.futureverse.org 38 | - text: globals 39 | href: https://globals.futureverse.org 40 | - text: listenv 41 | href: https://listenv.futureverse.org 42 | - text: parallelly 43 | href: https://parallelly.futureverse.org 44 | - text: progressr 45 | href: https://progressr.futureverse.org 46 | - text: BiocParallel.FutureParam (experimental) 47 | href: https://BiocParallel.FutureParam.futureverse.org 48 | - text: future.tools (experimental) 49 | href: https://future.tools.futureverse.org 50 | - text: future.mapreduce (experimental) 51 | href: https://future.mapreduce.futureverse.org 52 | - text: marshal (experimental) 53 | href: https://marshal.futureverse.org 54 | cran: 55 | icon: fab fa-r-project 56 | href: https://cloud.r-project.org/package=future.batchtools 57 | 58 | search: 59 | exclude: ['README_ja.md'] 60 | 61 | template: 62 | params: 63 | docsearch: 64 | api_key: aa6e02fc501886fb0f7c91ac4e300456 65 | index_name: futureverse 66 | algoliaOptions: { 'facetFilters': ['project:future.batchtools'] } 67 | ganalytics: G-SB3EQSD9FR 68 | bootstrap: 5 69 | light-switch: true 70 | -------------------------------------------------------------------------------- /pkgdown/_pkgdown.yml.rsp: -------------------------------------------------------------------------------- 1 | <% 2 | pkgs_mapreduce <- c("future.apply", "doFuture", "furrr") 3 | pkgs_backend <- c("future.batchtools", "future.callr", "future.mirai") 4 | pkgs <- c("globals", "listenv", "parallelly", "future", "future.tests", "progressr", pkgs_mapreduce, pkgs_backend) 5 | pkgs_extra <- c("BiocParallel.FutureParam", "future.tools", "future.mapreduce", "marshal") 6 | pkgs <- c(sort(pkgs), pkgs_extra) 7 | urls <- sprintf("https://%s.futureverse.org", pkgs) 8 | names(urls) <- pkgs 9 | 10 | file <- file.path(c(".", ".."), "DESCRIPTION") 11 | file <- file[utils::file_test("-f", file)] 12 | pkg <- read.dcf(file)[,"Package"] 13 | 14 | #common_support <- c("future", "future.apply", "future.callr", "future.batchtools", "doFuture", "BiocParallel.FutureParam") 15 | %> 16 | url: https://<%= pkg %>.futureverse.org 17 | 18 | home: 19 | links: 20 | - text: Roadmap/Milestones 21 | href: https://github.com/<%= gsub("(^.*:|[.]git$)", "", subset(gert::git_remote_list(), name == "origin")$url) %>/milestones 22 | - text: The Futureverse Project 23 | href: https://www.futureverse.org/ 24 | - text: Futureverse User Forum 25 | href: https://github.com/futureverse/future/discussions 26 | 27 | navbar: 28 | structure: 29 | right: [search, futureverse, pkgs, cran, github, lightswitch] 30 | components: 31 | futureverse: 32 | icon: fas fa-home 33 | href: https://www.futureverse.org/ 34 | pkgs: 35 | text: Packages 36 | menu: 37 | <% for (name in names(urls)) { %> 38 | - text: <%= name %> <% if (name %in% pkgs_extra) { %>(experimental)<% } else if (name %in% pkgs_backend) { %>(backend)<% } else if (name %in% pkgs_mapreduce) { %>(map-reduce)<% } %> 39 | href: <%= urls[name] %> 40 | <% } %> 41 | cran: 42 | icon: fab fa-r-project 43 | href: https://cloud.r-project.org/package=<%= pkg %> 44 | 45 | search: 46 | exclude: ['README_ja.md'] 47 | 48 | template: 49 | params: 50 | docsearch: 51 | api_key: aa6e02fc501886fb0f7c91ac4e300456 52 | index_name: futureverse 53 | algoliaOptions: { 'facetFilters': ['project:<%= pkg %>'] } 54 | ganalytics: G-SB3EQSD9FR 55 | bootstrap: 5 56 | light-switch: true 57 | -------------------------------------------------------------------------------- /pkgdown/favicon/apple-touch-icon-120x120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futureverse/future.batchtools/a2435e263e91da4b9a77ad57ba21875be90f8785/pkgdown/favicon/apple-touch-icon-120x120.png -------------------------------------------------------------------------------- /pkgdown/favicon/apple-touch-icon-60x60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futureverse/future.batchtools/a2435e263e91da4b9a77ad57ba21875be90f8785/pkgdown/favicon/apple-touch-icon-60x60.png -------------------------------------------------------------------------------- /pkgdown/favicon/apple-touch-icon-76x76.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futureverse/future.batchtools/a2435e263e91da4b9a77ad57ba21875be90f8785/pkgdown/favicon/apple-touch-icon-76x76.png -------------------------------------------------------------------------------- /pkgdown/favicon/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futureverse/future.batchtools/a2435e263e91da4b9a77ad57ba21875be90f8785/pkgdown/favicon/apple-touch-icon.png -------------------------------------------------------------------------------- /pkgdown/favicon/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futureverse/future.batchtools/a2435e263e91da4b9a77ad57ba21875be90f8785/pkgdown/favicon/favicon-16x16.png -------------------------------------------------------------------------------- /pkgdown/favicon/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futureverse/future.batchtools/a2435e263e91da4b9a77ad57ba21875be90f8785/pkgdown/favicon/favicon-32x32.png -------------------------------------------------------------------------------- /pkgdown/favicon/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futureverse/future.batchtools/a2435e263e91da4b9a77ad57ba21875be90f8785/pkgdown/favicon/favicon.ico -------------------------------------------------------------------------------- /revdep/README.md: -------------------------------------------------------------------------------- 1 | # Platform 2 | 3 | |field |value | 4 | |:--------|:---------------------------------------------------------| 5 | |version |R version 4.5.0 (2025-04-11) | 6 | |os |Rocky Linux 8.10 (Green Obsidian) | 7 | |system |x86_64, linux-gnu | 8 | |ui |X11 | 9 | |language |en | 10 | |collate |en_US.UTF-8 | 11 | |ctype |en_US.UTF-8 | 12 | |tz |America/Los_Angeles | 13 | |date |2025-06-05 | 14 | |pandoc |3.6.3 @ /software/c4/cbi/software/pandoc-3.6.3/bin/pandoc | 15 | |quarto |NA | 16 | 17 | # Dependencies 18 | 19 | |package |old |new |Δ | 20 | |:-----------------|:------|:-----------|:--| 21 | |future.batchtools |0.12.1 |0.12.1-9010 |* | 22 | |backports |1.5.0 |1.5.0 | | 23 | |base64url |1.4 |1.4 | | 24 | |batchtools |0.9.17 |0.9.17 | | 25 | |brew |1.0-10 |1.0-10 | | 26 | |checkmate |2.3.2 |2.3.2 | | 27 | |cli |3.6.5 |3.6.5 | | 28 | |codetools |0.2-20 |0.2-20 | | 29 | |crayon |1.5.3 |1.5.3 | | 30 | |data.table |1.17.4 |1.17.4 | | 31 | |digest |0.6.37 |0.6.37 | | 32 | |fs |1.6.6 |1.6.6 | | 33 | |future |1.58.0 |1.58.0 | | 34 | |globals |0.18.0 |0.18.0 | | 35 | |glue |1.8.0 |1.8.0 | | 36 | |hms |1.1.3 |1.1.3 | | 37 | |lifecycle |1.0.4 |1.0.4 | | 38 | |listenv |0.9.1 |0.9.1 | | 39 | |parallelly |1.45.0 |1.45.0 | | 40 | |pkgconfig |2.0.3 |2.0.3 | | 41 | |prettyunits |1.2.0 |1.2.0 | | 42 | |progress |1.2.3 |1.2.3 | | 43 | |R6 |2.6.1 |2.6.1 | | 44 | |rappdirs |0.3.3 |0.3.3 | | 45 | |rlang |1.1.6 |1.1.6 | | 46 | |stringi |1.8.7 |1.8.7 | | 47 | |vctrs |0.6.5 |0.6.5 | | 48 | |withr |3.0.2 |3.0.2 | | 49 | 50 | # Revdeps 51 | 52 | ## All (6) 53 | 54 | |package |version |error |warning |note | 55 | |:-----------|:-------|:-----|:-------|:----| 56 | |[batchtools](problems.md#batchtools)|0.9.17 | | |1 | 57 | |FAMoS |0.3.1 | | | | 58 | |futureverse |0.1.0 | | | | 59 | |nncc |2.0.0 | | | | 60 | |SimDesign |2.19.2 | | | | 61 | |targets |1.11.3 | | | | 62 | 63 | -------------------------------------------------------------------------------- /revdep/cran.md: -------------------------------------------------------------------------------- 1 | ## revdepcheck results 2 | 3 | We checked 6 reverse dependencies, comparing R CMD check results across CRAN and dev versions of this package. 4 | 5 | * We saw 0 new problems 6 | * We failed to check 0 packages 7 | 8 | -------------------------------------------------------------------------------- /revdep/failures.md: -------------------------------------------------------------------------------- 1 | *Wow, no problems at all. :)* -------------------------------------------------------------------------------- /revdep/problems.md: -------------------------------------------------------------------------------- 1 | # batchtools 2 | 3 |
4 | 5 | * Version: 0.9.17 6 | * GitHub: https://github.com/mllg/batchtools 7 | * Source code: https://github.com/cran/batchtools 8 | * Date/Publication: 2023-04-20 14:20:06 UTC 9 | * Number of recursive dependencies: 81 10 | 11 | Run `revdepcheck::revdep_details(, "batchtools")` for more info 12 | 13 |
14 | 15 | ## In both 16 | 17 | * checking Rd files ... NOTE 18 | ``` 19 | checkRd: (-1) JobCollection.Rd:41: Lost braces 20 | 41 | \item{namespaces}{code{character} with required packages to load via \code{\link[base]{requireNamespace}}.} 21 | | ^ 22 | checkRd: (-1) addAlgorithm.Rd:31: Lost braces 23 | 31 | Algorithms are functions which get the code{data} part as well as the problem instance (the return value of the 24 | | ^ 25 | ``` 26 | 27 | -------------------------------------------------------------------------------- /revdep/revdepcheck.Renviron: -------------------------------------------------------------------------------- 1 | ## Environment variables set by revdepcheck.extras::run() 2 | R_REVDEPCHECK_TIMEOUT=${R_REVDEPCHECK_TIMEOUT:-180} 3 | TAR_SKIP_CLUSTERMQ=${TAR_SKIP_CLUSTERMQ-true} 4 | 5 | -------------------------------------------------------------------------------- /revdep/run.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env Rscript 2 | 3 | precheck <- function() { 4 | ## WORKAROUND: Remove checked pkgs that use file links, which otherwise 5 | ## produce warnings which are promoted to errors by revdepcheck. 6 | unlink("revdep/checks/aroma.affymetrix", recursive = TRUE) 7 | } 8 | 9 | revdepcheck.extras::run() 10 | -------------------------------------------------------------------------------- /revdep/run.sge: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## Example: qsub -pe smp 24 -l h_rt=08:00:00 revdep/run.sge 3 | #$ -S /bin/bash 4 | #$ -R yes # SGE host reservation, highly recommended 5 | #$ -cwd # Current working directory 6 | #$ -j y # Join STDERR and STDOUT 7 | #$ -l mem_free=3G # On average 4 GiB of RAM per core (=a package check) 8 | #$ -m bea # email when job (b)egins, (e)nds, or (a)borts 9 | # 10 | # Pass on R-specific environment variables, iff set: 11 | ##$ -v _R_CHECK_LENGTH_1_CONDITION_ 12 | ##$ -v _R_CHECK_LENGTH_1_LOGIC2_ 13 | ##$ -v R_STARTUP_DEBUG 14 | 15 | ## SPECIAL: On Wynton HPC 16 | if [[ $SGE_CLUSTER_NAME == *wynton* ]]; then 17 | module load CBI r 18 | 19 | ## Install all packages to toward $TMPDIR, if revdep/library doesn't already exist. 20 | ## This will avoid some of the slowness on the global file system 21 | if [[ ! -d revdep/library ]]; then 22 | tmpdir=$(mktemp -d) 23 | ln -fs "$tmpdir" revdep/library 24 | [[ -d revdep/library ]] || { >&2 echo "ERROR: Failed to link revdep/library/ to $tmpdir"; exit 1; } 25 | fi 26 | 27 | ## To check in on revdep/library/ on the running host (see below), submit a job like: 28 | ## echo "ls -lR revdep/library/" | qsub -cwd -j yes -l hostname= 29 | 30 | ## Assert that revdep/library is on $TMPDIR 31 | #if [[ ! "$(readlink revdep/library)" = $TMPDIR* ]]; then 32 | # >&2 echo "ERROR: revdep/library/ already exists but is not on $TMPDIR" 33 | # exit 1 34 | #fi 35 | fi 36 | 37 | echo "HOSTNAME: $HOSTNAME" 38 | ls -l revdep/ 39 | 40 | Rscript --version 41 | Rscript -e ".libPaths()" 42 | Rscript revdep/run.R 43 | -------------------------------------------------------------------------------- /tests/BatchtoolsFuture,gc.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | message("*** BatchtoolsFuture() and garbage collection ...") 4 | 5 | plan(batchtools_local) 6 | 7 | for (how in c("resolve", "value")) { 8 | f <- future({ 1 }) 9 | 10 | if (how == "value") { 11 | v <- value(f) 12 | print(v) 13 | } else if (how == "resolve") { 14 | resolve(f) 15 | } 16 | 17 | stopifnot(resolved(f)) 18 | 19 | reg <- f$config$reg 20 | 21 | ## Force removal of batchtools registry files 22 | rm(list = "f") 23 | gc() 24 | 25 | ## Assert removal of files only happens if there was not 26 | ## a failure and option future.delete is not TRUE. 27 | stopifnot(!file_test("-d", reg$file.dir)) 28 | fail <- try(checkIds(reg, ids = 1L), silent = TRUE) 29 | stopifnot(inherits(fail, "try-error")) 30 | } ## for (how ...) 31 | 32 | 33 | message("*** BatchtoolsFuture() and garbage collection ... DONE") 34 | 35 | source("incl/end.R") 36 | -------------------------------------------------------------------------------- /tests/BatchtoolsFuture.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | message("*** BatchtoolsFuture() ...") 4 | 5 | message("*** BatchtoolsFuture() - cleanup ...") 6 | 7 | f <- batchtools_local({ 1L }) 8 | res <- await(f, cleanup = TRUE) 9 | print(res) 10 | stopifnot(res$value == 1L) 11 | 12 | message("*** BatchtoolsFuture() - cleanup ... DONE") 13 | 14 | 15 | message("*** BatchtoolsFuture() - deleting exceptions ...") 16 | 17 | ## Printing a deleted future 18 | f <- batchtools_local(42L) 19 | print(f) 20 | v <- value(f) 21 | print(v) 22 | stopifnot(v == 42L) 23 | res <- delete(f) 24 | print(f) 25 | res <- delete(f) 26 | print(f) 27 | 28 | message("*** BatchtoolsFuture() - deleting exceptions ... DONE") 29 | 30 | 31 | message("*** BatchtoolsFuture() - registry exceptions ...") 32 | 33 | ## Non-existing batchtools registry 34 | f <- BatchtoolsFuture({ x <- 1 }) 35 | 36 | ## Hack to emulate where batchtools registry is deleted or fails 37 | f$state <- "running" 38 | if (!is.null(f$config$reg)) { 39 | path <- f$config$reg$file.dir 40 | unlink(path, recursive = TRUE) 41 | } 42 | 43 | res <- tryCatch({ 44 | value(f) 45 | }, error = function(ex) ex) 46 | print(res) 47 | stopifnot(inherits(res, "error")) 48 | 49 | res <- tryCatch({ 50 | await(f) 51 | }, error = function(ex) ex) 52 | print(res) 53 | stopifnot(inherits(res, "error")) 54 | 55 | 56 | message("*** BatchtoolsFuture() - registry exceptions ... DONE") 57 | 58 | message("*** BatchtoolsFuture() - exceptions ...") 59 | 60 | res <- try(f <- BatchtoolsFuture(42L, workers = integer(0)), silent = TRUE) 61 | print(res) 62 | stopifnot(inherits(res, "try-error")) 63 | 64 | res <- try(f <- BatchtoolsFuture(42L, workers = 0L), silent = TRUE) 65 | print(res) 66 | stopifnot(inherits(res, "try-error")) 67 | 68 | res <- try(f <- BatchtoolsFuture(42L, workers = TRUE), silent = TRUE) 69 | print(res) 70 | stopifnot(inherits(res, "try-error")) 71 | 72 | message("*** BatchtoolsFuture() - exceptions ... DONE") 73 | 74 | 75 | message("*** BatchtoolsFuture() - timeout ...") 76 | 77 | if (fullTest && availableCores(constraints = "multicore") > 1) { 78 | plan(batchtools_multicore) 79 | 80 | options(future.wait.timeout = 0.15, future.wait.interval = 0.1) 81 | 82 | f <- future({ 83 | Sys.sleep(5) 84 | x <- 1 85 | }) 86 | 87 | res <- tryCatch({ 88 | value(f) 89 | }, error = function(ex) ex) 90 | stopifnot(inherits(res, "error")) 91 | } 92 | 93 | 94 | message("*** BatchtoolsFuture() - timeout ... DONE") 95 | 96 | 97 | 98 | message("*** BatchtoolsFuture() ... DONE") 99 | 100 | source("incl/end.R") 101 | -------------------------------------------------------------------------------- /tests/BatchtoolsFutureError.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | message("*** BatchtoolsFutureError() ...") 4 | 5 | plan(batchtools_local) 6 | 7 | for (cleanup in c(FALSE, TRUE)) { 8 | mprintf("*** batchtools future error w/ future.delete = %s ...\n", cleanup) 9 | 10 | options(future.delete = cleanup) 11 | 12 | f <- future({ 13 | x <- 1 14 | print(x) 15 | stop("Woops!") 16 | }) 17 | 18 | resolve(f) 19 | 20 | ## Assert future is listed as resolved 21 | stopifnot(resolved(f)) 22 | 23 | reg <- f$config$reg 24 | ## Force garbage collection of future which will possibly 25 | ## result in the removal of batchtools registry files 26 | 27 | reg.finalizer(f, function(f) { 28 | message("Garbage collecting future ...") 29 | print(f) 30 | message("Garbage collecting future ... DONE") 31 | }, onexit = TRUE) 32 | rm(list = "f") 33 | gc() 34 | message(" - Future removed and garbage collected.") 35 | mprintf(" - batchtools Registry path (%s) exists: %s\n", 36 | sQuote(reg$file.dir), file_test("-d", reg$file.dir)) 37 | 38 | ## Assert removal of files only happens if there was not 39 | ## a failure and option future.delete is not TRUE. 40 | if (!cleanup) { 41 | ## FIXME: Does the new future::FutureResult trigger garbage collection? 42 | stopifnot(file_test("-d", reg$file.dir)) 43 | log <- batchtools::getLog(reg = reg, id = 1L) 44 | print(log) 45 | 46 | ## Now manually delete batchtools Registry 47 | batchtools::removeRegistry(wait = 0.0, reg = reg) 48 | } 49 | 50 | stopifnot(!file_test("-d", reg$file.dir)) 51 | 52 | mprintf("*** batchtools future error w/ future.delete = %s ... DONE\n", cleanup) 53 | } ## for (cleanup ...) 54 | 55 | 56 | message("*** BatchtoolsFuture - expired ...") 57 | plan(batchtools_local) 58 | msg <- "Abruptly terminating the future!" 59 | f <- future({ 60 | cat(file = stderr(), msg) 61 | quit(save = "no") 62 | }) 63 | res <- tryCatch({ 64 | v <- value(f) 65 | }, error = identity) 66 | stopifnot(inherits(res, "error"), 67 | inherits(res, "FutureError")) 68 | err_msg <- unlist(strsplit(conditionMessage(res), split = "\n", fixed = TRUE)) 69 | stopifnot(any(grepl(msg, err_msg, fixed = TRUE))) 70 | 71 | message("*** BatchtoolsFuture - expired ... done") 72 | 73 | 74 | if (fullTest) { 75 | message("*** BatchtoolsFuture - deleting running ...") 76 | 77 | plan(batchtools_multicore) 78 | 79 | f <- future({ 80 | Sys.sleep(2) 81 | 42L 82 | }) 83 | 84 | if (!resolved(f)) { 85 | res <- delete(f, onRunning = "skip") 86 | stopifnot(isTRUE(res)) 87 | } 88 | 89 | if (!resolved(f)) { 90 | res <- tryCatch({ 91 | delete(f, onRunning = "warning") 92 | }, warning = function(w) w) 93 | stopifnot(inherits(res, "warning")) 94 | } 95 | 96 | if (!resolved(f)) { 97 | res <- tryCatch({ 98 | delete(f, onRunning = "error") 99 | }, error = function(ex) ex) 100 | stopifnot(inherits(res, "error")) 101 | } 102 | 103 | message("*** BatchtoolsFuture - deleting running ... DONE") 104 | } ## if (fullTest) 105 | 106 | 107 | message("*** BatchtoolsFutureError() ... DONE") 108 | 109 | source("incl/end.R") 110 | -------------------------------------------------------------------------------- /tests/batchtools_custom.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | library("batchtools") 3 | library("listenv") 4 | 5 | message("*** batchtools_custom() ...") 6 | 7 | message("*** batchtools_custom() w/ 'conf.file' on R_BATCHTOOLS_SEARCH_PATH") 8 | 9 | f <- batchtools_custom({ 10 | 42L 11 | }) 12 | print(f) 13 | stopifnot(inherits(f, "BatchtoolsFuture")) 14 | v <- value(f) 15 | print(v) 16 | stopifnot(v == 42L) 17 | 18 | 19 | message("*** batchtools_custom() w/ 'cluster.functions' without globals") 20 | 21 | cf <- makeClusterFunctionsInteractive(external = TRUE) 22 | str(cf) 23 | 24 | f <- batchtools_custom({ 25 | 42L 26 | }, cluster.functions = cf) 27 | stopifnot(inherits(f, "BatchtoolsFuture")) 28 | 29 | ## Check whether a batchtools_custom future is resolved 30 | ## or not will force evaluation 31 | print(is_resolved <- resolved(f)) 32 | stopifnot(is_resolved) 33 | 34 | y <- value(f) 35 | print(y) 36 | stopifnot(y == 42L) 37 | 38 | 39 | message("*** batchtools_custom() w/ 'cluster.functions' with globals") 40 | ## A global variable 41 | a <- 0 42 | f <- batchtools_custom({ 43 | b <- 3 44 | c <- 2 45 | a * b * c 46 | }, cluster.functions = cf) 47 | print(f) 48 | 49 | ## Although 'f' is a batchtools_custom future and therefore 50 | ## resolved/evaluates the future expression only 51 | ## when the value is requested, any global variables 52 | ## identified in the expression (here 'a') are 53 | ## "frozen" at the time point when the future is 54 | ## created. Because of this, 'a' preserved the 55 | ## zero value although we reassign it below 56 | a <- 7 ## Make sure globals are frozen 57 | v <- value(f) 58 | print(v) 59 | stopifnot(v == 0) 60 | 61 | 62 | message("*** batchtools_custom() w/ 'cluster.functions' with globals (tricky)") 63 | x <- listenv() 64 | for (ii in 1:2) { 65 | x[[ii]] <- batchtools_custom({ ii }, globals = TRUE, cluster.functions = cf) 66 | } 67 | v <- unlist(value(x)) 68 | stopifnot(all(v == 1:2)) ## Make sure globals are frozen 69 | 70 | 71 | message("*** batchtools_custom() w/ 'cluster.functions' and errors") 72 | f <- batchtools_custom({ 73 | stop("Whoops!") 74 | 1 75 | }, cluster.functions = cf) 76 | v <- value(f, signal = FALSE) 77 | print(v) 78 | stopifnot(inherits(v, "simpleError")) 79 | 80 | res <- try({ v <- value(f) }, silent = TRUE) 81 | print(res) 82 | stopifnot(inherits(res, "try-error")) 83 | 84 | ## Error is repeated 85 | res <- try(value(f), silent = TRUE) 86 | print(res) 87 | stopifnot(inherits(res, "try-error")) 88 | 89 | message("*** batchtools_custom() ... DONE") 90 | 91 | source("incl/end.R") 92 | -------------------------------------------------------------------------------- /tests/batchtools_hpc.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | ## Setup all strategies including custom once for testing on HPC environments 4 | print(all_strategies()) 5 | 6 | message("All HPC strategies:") 7 | 8 | strategies <- c("batchtools_lsf", "batchtools_openlava", "batchtools_sge", 9 | "batchtools_slurm", "batchtools_torque") 10 | mprint(strategies, debug = TRUE) 11 | 12 | message("Supported HPC strategies:") 13 | strategies <- strategies[sapply(strategies, FUN = test_strategy)] 14 | mprint(strategies, debug = TRUE) 15 | 16 | for (strategy in strategies) { 17 | plan(strategy) 18 | print(plan()) 19 | 20 | f <- future(42L) 21 | print(f) 22 | v <- value(f) 23 | print(v) 24 | stopifnot(v == 42L) 25 | 26 | x %<-% Sys.info() 27 | print(x) 28 | 29 | message(sprintf("*** %s() ... DONE", strategy)) 30 | } 31 | 32 | source("incl/end.R") 33 | -------------------------------------------------------------------------------- /tests/batchtools_interactive.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | library("listenv") 3 | 4 | message("*** batchtools_interactive() ...") 5 | 6 | message("*** batchtools_interactive() without globals") 7 | 8 | f <- batchtools_interactive({ 9 | 42L 10 | }) 11 | stopifnot(inherits(f, "BatchtoolsFuture")) 12 | 13 | ## Check whether a batchtools_interactive future is resolved 14 | ## or not will force evaluation 15 | print(is_resolved <- resolved(f)) 16 | stopifnot(is_resolved) 17 | 18 | y <- value(f) 19 | print(y) 20 | stopifnot(y == 42L) 21 | 22 | 23 | message("*** batchtools_interactive() with globals") 24 | ## A global variable 25 | a <- 0 26 | f <- batchtools_interactive({ 27 | b <- 3 28 | c <- 2 29 | a * b * c 30 | }) 31 | 32 | ## Although 'f' is a batchtools_interactive future and therefore 33 | ## resolved/evaluates the future expression only 34 | ## when the value is requested, any global variables 35 | ## identified in the expression (here 'a') are 36 | ## "frozen" at the time point when the future is 37 | ## created. Because of this, 'a' preserved the 38 | ## zero value although we reassign it below 39 | a <- 7 ## Make sure globals are frozen 40 | v <- value(f) 41 | print(v) 42 | stopifnot(v == 0) 43 | 44 | 45 | message("*** batchtools_interactive() with globals (tricky)") 46 | x <- listenv() 47 | for (ii in 1:2) x[[ii]] <- batchtools_interactive({ ii }, globals = TRUE) 48 | v <- unlist(value(x)) 49 | stopifnot(all(v == 1:2)) ## Make sure globals are frozen 50 | 51 | 52 | message("*** batchtools_interactive() and errors") 53 | f <- batchtools_interactive({ 54 | stop("Whoops!") 55 | 1 56 | }) 57 | v <- value(f, signal = FALSE) 58 | print(v) 59 | stopifnot(inherits(v, "simpleError")) 60 | 61 | res <- try({ v <- value(f) }, silent = TRUE) 62 | print(res) 63 | stopifnot(inherits(res, "try-error")) 64 | 65 | ## Error is repeated 66 | res <- try(value(f), silent = TRUE) 67 | print(res) 68 | stopifnot(inherits(res, "try-error")) 69 | 70 | message("*** batchtools_interactive() ... DONE") 71 | 72 | source("incl/end.R") 73 | -------------------------------------------------------------------------------- /tests/batchtools_local.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | library("listenv") 3 | 4 | message("*** batchtools_local() ...") 5 | 6 | message("*** batchtools_local() without globals") 7 | 8 | f <- batchtools_local({ 9 | 42L 10 | }) 11 | stopifnot(inherits(f, "BatchtoolsFuture")) 12 | 13 | ## Check whether a batchtools_local future is resolved 14 | ## or not will force evaluation 15 | print(is_resolved <- resolved(f)) 16 | stopifnot(is_resolved) 17 | 18 | y <- value(f) 19 | print(y) 20 | stopifnot(y == 42L) 21 | 22 | 23 | message("*** batchtools_local() with globals") 24 | ## A global variable 25 | a <- 0 26 | f <- batchtools_local({ 27 | b <- 3 28 | c <- 2 29 | a * b * c 30 | }) 31 | 32 | ## Although 'f' is a batchtools_local future and therefore 33 | ## resolved/evaluates the future expression only 34 | ## when the value is requested, any global variables 35 | ## identified in the expression (here 'a') are 36 | ## "frozen" at the time point when the future is 37 | ## created. Because of this, 'a' preserved the 38 | ## zero value although we reassign it below 39 | a <- 7 ## Make sure globals are frozen 40 | v <- value(f) 41 | print(v) 42 | stopifnot(v == 0) 43 | 44 | 45 | message("*** batchtools_local() with globals (tricky)") 46 | x <- listenv() 47 | for (ii in 1:2) x[[ii]] <- batchtools_local({ ii }, globals = TRUE) 48 | v <- unlist(value(x)) 49 | stopifnot(all(v == 1:2)) ## Make sure globals are frozen 50 | 51 | 52 | message("*** batchtools_local() and errors") 53 | f <- batchtools_local({ 54 | stop("Whoops!") 55 | 1 56 | }) 57 | v <- value(f, signal = FALSE) 58 | print(v) 59 | stopifnot(inherits(v, "simpleError")) 60 | 61 | res <- try({ v <- value(f) }, silent = TRUE) 62 | print(res) 63 | stopifnot(inherits(res, "try-error")) 64 | 65 | ## Error is repeated 66 | res <- try(value(f), silent = TRUE) 67 | print(res) 68 | stopifnot(inherits(res, "try-error")) 69 | 70 | message("*** batchtools_local() ... DONE") 71 | 72 | source("incl/end.R") 73 | -------------------------------------------------------------------------------- /tests/batchtools_multicore.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | library("listenv") 3 | 4 | message("*** batchtools_multicore() ...") 5 | 6 | for (cores in 1:min(2L, availableCores("multicore"))) { 7 | ## FIXME: 8 | if (!fullTest && cores > 1) next 9 | 10 | ## CRAN processing times: 11 | ## On Windows 32-bit, don't run these tests 12 | if (!fullTest && isWin32) next 13 | 14 | mprintf("Testing with %d cores ...\n", cores) 15 | options(mc.cores = cores - 1L) 16 | 17 | if (!supportsMulticore()) { 18 | mprintf("batchtools multicore futures are not supporting on '%s'. Falling back to use synchroneous batchtools local futures\n", .Platform$OS.type) #nolint 19 | } 20 | 21 | for (globals in c(FALSE, TRUE)) { 22 | mprintf("*** batchtools_multicore(..., globals = %s) without globals\n", 23 | globals) 24 | 25 | f <- batchtools_multicore({ 26 | 42L 27 | }, globals = globals) 28 | stopifnot( 29 | inherits(f, "BatchtoolsFuture") || 30 | ((cores == 1 || !supportsMulticore()) && inherits(f, "SequentialFuture")) 31 | ) 32 | 33 | print(resolved(f)) 34 | y <- value(f) 35 | print(y) 36 | stopifnot(y == 42L) 37 | 38 | mprintf("*** batchtools_multicore(..., globals = %s) with globals\n", 39 | globals) 40 | ## A global variable 41 | a <- 0 42 | f <- batchtools_multicore({ 43 | b <- 3 44 | c <- 2 45 | a * b * c 46 | }, globals = globals) 47 | 48 | ## A multicore future is evaluated in a separated 49 | ## forked process. Changing the value of a global 50 | ## variable should not affect the result of the 51 | ## future. 52 | a <- 7 ## Make sure globals are frozen 53 | if (globals || f$config$reg$cluster.functions$name == "Multicore") { 54 | v <- value(f) 55 | print(v) 56 | stopifnot(v == 0) 57 | } else { 58 | res <- tryCatch({ value(f) }, error = identity) 59 | print(res) 60 | stopifnot(inherits(res, "simpleError")) 61 | } 62 | 63 | 64 | mprintf("*** batchtools_multicore(..., globals = %s) with globals and blocking\n", globals) #nolint 65 | x <- listenv() 66 | for (ii in 1:2) { 67 | mprintf(" - Creating batchtools_multicore future #%d ...\n", ii) 68 | x[[ii]] <- batchtools_multicore({ ii }, globals = globals) 69 | } 70 | mprintf(" - Resolving %d batchtools_multicore futures\n", length(x)) 71 | if (globals || f$config$reg$cluster.functions$name == "Multicore") { 72 | v <- unlist(value(x)) 73 | stopifnot(all(v == 1:2)) 74 | } else { 75 | v <- lapply(x, FUN = function(f) tryCatch(value(f), error = identity)) 76 | stopifnot(all(sapply(v, FUN = inherits, "simpleError"))) 77 | } 78 | } # for (globals ...) 79 | 80 | if (cores > 1) { 81 | message("*** batchtools_multicore(..., workers = 1L) ...") 82 | 83 | a <- 2 84 | b <- 3 85 | y_truth <- a * b 86 | 87 | f <- batchtools_multicore({ a * b }, workers = 1L) 88 | rm(list = c("a", "b")) 89 | 90 | v <- value(f) 91 | print(v) 92 | stopifnot(v == y_truth) 93 | 94 | message("*** batchtools_multicore(..., workers = 1L) ... DONE") 95 | } 96 | 97 | mprintf("Testing with %d cores ... DONE\n", cores) 98 | } ## for (cores ...) 99 | 100 | 101 | ## CRAN processing times: 102 | ## On Windows 32-bit, don't run these tests 103 | if (fullTest || !isWin32) { 104 | mprintf("*** batchtools_multicore() and errors\n") 105 | f <- batchtools_multicore({ 106 | stop("Whoops!") 107 | 1 108 | }) 109 | v <- value(f, signal = FALSE) 110 | print(v) 111 | stopifnot(inherits(v, "simpleError")) 112 | 113 | res <- try(value(f), silent = TRUE) 114 | print(res) 115 | stopifnot(inherits(res, "try-error")) 116 | 117 | ## Error is repeated 118 | res <- try(value(f), silent = TRUE) 119 | print(res) 120 | stopifnot(inherits(res, "try-error")) 121 | } 122 | 123 | message("*** batchtools_multicore() ... DONE") 124 | 125 | source("incl/end.R") 126 | -------------------------------------------------------------------------------- /tests/batchtools_ssh.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | library("listenv") 3 | 4 | message("*** batchtools_ssh() ...") 5 | 6 | plan(batchtools_ssh, workers = 2L) 7 | supports_ssh <- tryCatch({ 8 | f <- future(42L) 9 | v <- value(f) 10 | identical(v, 42L) 11 | }, error = function(e) FALSE) 12 | message("Supports batchtools_ssh: ", supports_ssh) 13 | 14 | if (supports_ssh) { 15 | message("future(a) ...") 16 | a0 <- a <- 42 17 | f <- future(a) 18 | stopifnot(identical(f$globals$a, a0)) 19 | v <- value(f) 20 | stopifnot(identical(v, a0)) 21 | 22 | message("future(a, lazy = TRUE) ...") 23 | a0 <- a <- 42 24 | f <- future(a, lazy = TRUE) 25 | rm(list = "a") 26 | stopifnot(identical(f$globals$a, a0)) 27 | v <- value(f) 28 | stopifnot(identical(v, a0)) 29 | } ## if (supports_ssh) 30 | 31 | message("*** batchtools_ssh() ... DONE") 32 | 33 | source("incl/end.R") 34 | -------------------------------------------------------------------------------- /tests/batchtools_template.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | message("*** batchtools_template() ...") 4 | 5 | ## NOTE: Here we use invalid 'workers = FALSE' in order to 6 | ## prevent the batchtools future from actually starting, 7 | ## because we cannot assume that system has these schedulers. 8 | ## NOTE: Some of them will give an earlier error because 9 | ## no default template file was found. 10 | res <- try(batchtools_lsf({ 42L }, workers = FALSE)) 11 | stopifnot(inherits(res, "try-error")) 12 | 13 | res <- try(batchtools_openlava({ 42L }, workers = FALSE)) 14 | stopifnot(inherits(res, "try-error")) 15 | 16 | res <- try(batchtools_sge({ 42L }, workers = FALSE)) 17 | stopifnot(inherits(res, "try-error")) 18 | 19 | res <- try(batchtools_slurm({ 42L }, workers = FALSE)) 20 | stopifnot(inherits(res, "try-error")) 21 | 22 | res <- try(batchtools_torque({ 42L }, workers = FALSE)) 23 | stopifnot(inherits(res, "try-error")) 24 | 25 | message("*** batchtools_template() ... DONE") 26 | 27 | source("incl/end.R") 28 | -------------------------------------------------------------------------------- /tests/demo.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | plan(batchtools_local) 4 | 5 | ## CRAN processing times: 6 | ## On Windows 32-bit, don't run these tests via batchtools 7 | if (!fullTest && isWin32) plan(sequential) 8 | 9 | options(future.demo.mandelbrot.nrow = 2L) 10 | options(future.demo.mandelbrot.resolution = 50L) 11 | options(future.demo.mandelbrot.delay = FALSE) 12 | 13 | message("*** Demos ...") 14 | 15 | message("*** Mandelbrot demo of the 'future' package ...") 16 | 17 | demo("mandelbrot", package = "future", ask = FALSE) 18 | 19 | message("*** Demos ... DONE") 20 | 21 | source("incl/end.R") 22 | -------------------------------------------------------------------------------- /tests/dotdotdot.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | library("listenv") 3 | 4 | strategies <- c("batchtools_interactive", "batchtools_local") 5 | 6 | ## CRAN processing times: 7 | ## On Windows 32-bit, don't run these tests 8 | if (!fullTest && isWin32) strategies <- character(0L) 9 | 10 | 11 | message("*** Global argument '...' in futures ...") 12 | 13 | sum_fcns <- list() 14 | 15 | sum_fcns$A <- function(x, ...) { 16 | message("Arguments '...' exists: ", exists("...", inherits = TRUE)) 17 | y %<-% { sum(x, ...) } 18 | y 19 | } 20 | 21 | 22 | sum_fcns$B <- function(x, ...) { 23 | sumt <- function(x) { 24 | message("Arguments '...' exists: ", exists("...", inherits = TRUE)) 25 | y %<-% { sum(x, ...) } 26 | y 27 | } 28 | sumt(x) 29 | } 30 | 31 | sum_fcns$C <- function(x, y) { 32 | message("Arguments '...' exists: ", exists("...", inherits = TRUE)) 33 | y %<-% { sum(x, y) } 34 | y 35 | } 36 | 37 | sum_fcns$D <- function(x, y) { 38 | message("Arguments '...' exists: ", exists("...", inherits = TRUE)) 39 | y %<-% { sum(x, y, ...) } 40 | y 41 | } 42 | 43 | 44 | for (strategy in strategies) { 45 | plan(strategy, substitute = FALSE) 46 | 47 | for (name in names(sum_fcns)) { 48 | mprintf("** Sum function '%s' with plan('%s') ...\n", name, strategy) 49 | sum_fcn <- sum_fcns[[name]] 50 | print(sum_fcn) 51 | y <- try(sum_fcn(1:2, 3)) 52 | print(y) 53 | if (name %in% c("D")) { 54 | stopifnot(inherits(y, "try-error")) 55 | } else { 56 | stopifnot(y == 6) 57 | } 58 | } 59 | } 60 | 61 | message("*** Global argument '...' in futures ... DONE") 62 | 63 | source("incl/end.R") 64 | -------------------------------------------------------------------------------- /tests/future,labels.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | message("*** Futures - labels ...") 4 | 5 | strategies <- c("batchtools_local") 6 | 7 | ## CRAN processing times: 8 | ## On Windows 32-bit, don't run these tests 9 | if (!fullTest && isWin32) strategies <- character(0L) 10 | 11 | for (strategy in strategies) { 12 | mprintf("- plan('%s') ...\n", strategy) 13 | plan(strategy) 14 | 15 | for (label in list(NULL, sprintf("strategy_%s", strategy))) { 16 | fcn <- get(strategy, mode = "function") 17 | stopifnot(inherits(fcn, strategy)) 18 | f <- fcn(42, label = label) 19 | stopifnot(identical(f$label, label)) 20 | v <- value(f) 21 | stopifnot(v == 42) 22 | print(f) 23 | 24 | f <- future(42, label = label) 25 | stopifnot(identical(f$label, label)) 26 | v <- value(f) 27 | stopifnot(v == 42) 28 | 29 | v %<-% { 42 } %label% label 30 | f <- futureOf(v) 31 | stopifnot(identical(f$label, label)) 32 | stopifnot(v == 42) 33 | 34 | } ## for (label ...) 35 | 36 | mprintf("- plan('%s') ... DONE\n", strategy) 37 | } ## for (strategy ...) 38 | 39 | message("*** Futures - labels ... DONE") 40 | 41 | source("incl/end.R") 42 | -------------------------------------------------------------------------------- /tests/future,lazy.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | message("*** Futures - lazy ...") 4 | 5 | strategies <- c("batchtools_local") 6 | 7 | ## CRAN processing times: 8 | ## On Windows 32-bit, don't run these tests 9 | if (!fullTest && isWin32) strategies <- character(0L) 10 | 11 | for (strategy in strategies) { 12 | mprintf("- plan('%s') ...\n", strategy) 13 | plan(strategy) 14 | 15 | a <- 42 16 | f <- future(2 * a, lazy = TRUE) 17 | a <- 21 18 | ## In future (> 1.14.0), resolved() will launch lazy future, 19 | ## which means for some backends (e.g. sequential) this means 20 | ## that resolved() might end up returning TRUE. 21 | f <- resolve(f) 22 | stopifnot(resolved(f)) 23 | v <- value(f) 24 | stopifnot(v == 84) 25 | 26 | a <- 42 27 | v %<-% { 2 * a } %lazy% TRUE 28 | a <- 21 29 | f <- futureOf(v) 30 | ## In future (> 1.14.0), resolved() will launch lazy future, 31 | ## which means for some backends (e.g. sequential) this means 32 | ## that resolved() might end up returning TRUE. 33 | f <- resolve(f) 34 | stopifnot(resolved(f)) 35 | stopifnot(v == 84) 36 | 37 | mprintf("- plan('%s') ... DONE\n", strategy) 38 | } ## for (strategy ...) 39 | 40 | message("*** Futures - lazy ... DONE") 41 | 42 | source("incl/end.R") 43 | -------------------------------------------------------------------------------- /tests/globals,formulas.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | library("datasets") ## cars data set 4 | library("stats") ## lm(), poly(), xtabs() 5 | 6 | plan(batchtools_local) 7 | 8 | ## CRAN processing times: 9 | ## On Windows 32-bit, don't run these tests on batchtools 10 | if (!fullTest && isWin32) plan(sequential) 11 | 12 | message("*** Globals - formulas ...") 13 | 14 | message("*** Globals - lm() ...") 15 | 16 | ## From example("lm", package = "stats") 17 | ctl <- c(4.17, 5.58, 5.18, 6.11, 4.50, 4.61, 5.17, 4.53, 5.33, 5.14) 18 | trt <- c(4.81, 4.17, 4.41, 3.59, 5.87, 3.83, 6.03, 4.89, 4.32, 4.69) 19 | group <- gl(2, 10, 20, labels = c("Ctl", "Trt")) 20 | weight <- c(ctl, trt) 21 | 22 | ## Truth: 23 | fit0 <- lm(weight ~ group - 1) 24 | print(fit0) 25 | 26 | ## Explicit future 27 | f <- future({ lm(weight ~ group - 1) }) 28 | fit <- value(f) 29 | print(fit) 30 | stopifnot(all.equal(fit, fit0)) 31 | 32 | ## Future assignment 33 | fit %<-% { lm(weight ~ group - 1) } 34 | print(fit) 35 | stopifnot(all.equal(fit, fit0)) 36 | 37 | message("*** Globals - lm() ... DONE") 38 | 39 | 40 | message("*** Globals - one-side formulas, e.g. xtabs(~ x) ...") 41 | 42 | x <- c(1, 1, 2, 2, 2) 43 | 44 | ## Truth: 45 | tbl0 <- xtabs(~ x) 46 | print(tbl0) 47 | 48 | ## Explicit future 49 | f <- future({ xtabs(~ x) }) 50 | tbl <- value(f) 51 | print(tbl) 52 | stopifnot(all.equal(tbl, tbl0)) 53 | 54 | ## Future assignment 55 | tbl %<-% { xtabs(~ x) } 56 | print(tbl) 57 | stopifnot(all.equal(tbl, tbl0)) 58 | 59 | message("*** Globals - one-side formulas, e.g. xtabs(~ x) ... DONE") 60 | 61 | 62 | message("*** Globals - lm(, data = cars) ...") 63 | 64 | exprs <- list( 65 | # "remove-intercept-term" form of no-intercept 66 | a = substitute({ lm(dist ~ . - 1, data = cars) }), 67 | # "make-intercept-zero" form of no-intercept 68 | b = substitute({ lm(dist ~ . + 0, data = cars) }), 69 | # doesn't do what we want here 70 | c = substitute({ lm(dist ~ speed + speed ^ 2, data = cars) }), 71 | # gets us a quadratic term 72 | d = substitute({ lm(dist ~ speed + I(speed ^ 2), data = cars) }), 73 | # avoid potential multicollinearity 74 | e = substitute({ lm(dist ~ poly(speed, 2), data = cars) }) 75 | ) 76 | 77 | for (kk in seq_along(exprs)) { 78 | expr <- exprs[[kk]] 79 | name <- names(exprs)[kk] 80 | mprintf("- Globals - lm(, data = cars) ...\n", 81 | kk, sQuote(name)) 82 | 83 | fit0 <- eval(expr) 84 | print(fit0) 85 | 86 | f <- future(expr, substitute = FALSE) 87 | fit <- value(f) 88 | print(fit) 89 | 90 | stopifnot(all.equal(fit, fit0)) 91 | } ## for (kk ...) 92 | 93 | message("*** Globals - lm(, data = cars) ... DONE") 94 | 95 | 96 | message("*** Globals - map(x, ~ expr) ...") 97 | 98 | ## A fake purrr::map() function with limited functionality 99 | map <- function(.x, .f, ...) { 100 | if (inherits(.f, "formula")) { 101 | expr <- .f[[-1]] 102 | .f <- eval(bquote(function(...) { 103 | .(expr) 104 | })) 105 | } 106 | eval(lapply(.x, FUN = .f, ...)) 107 | } 108 | 109 | inner_function <- function(x) { x + 1 } 110 | 111 | outer_function <- function(x) { 112 | map(1:2, ~ inner_function(.x)) 113 | } 114 | 115 | y0 <- outer_function(1L) 116 | str(y0) 117 | 118 | f <- future({ outer_function(1L) }) 119 | y <- value(f) 120 | str(y) 121 | stopifnot(all.equal(y, y0)) 122 | 123 | y %<-% { outer_function(1L) } 124 | str(y) 125 | stopifnot(all.equal(y, y0)) 126 | 127 | message("*** Globals - map(x, ~ expr) ... DONE") 128 | 129 | 130 | message("*** Globals - formulas ... DONE") 131 | 132 | source("incl/end.R") 133 | -------------------------------------------------------------------------------- /tests/globals,manual.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | plan(batchtools_local) 4 | 5 | ## CRAN processing times: 6 | ## On Windows 32-bit, don't run these tests on batchtools 7 | if (!fullTest && isWin32) plan(sequential) 8 | 9 | message("*** Globals - manually ...") 10 | 11 | message("*** Globals manually specified as named list ...") 12 | 13 | globals <- list( 14 | a = 1, 15 | b = 2, 16 | sumtwo = function(x) x[1] + x[2] 17 | ) 18 | 19 | ## Assign 'globals' globally 20 | attach_locally(globals) 21 | 22 | ## Truth 23 | v0 <- local({ 24 | x <- 1:10 25 | sumtwo(a + b * x) 26 | }) 27 | 28 | 29 | message("*** Globals - automatic ...") 30 | 31 | attach_locally(globals) 32 | f <- future({ 33 | x <- 1:10 34 | sumtwo(a + b * x) 35 | }, globals = TRUE) 36 | rm(list = names(globals)) 37 | y <- value(f) 38 | print(y) 39 | stopifnot(all.equal(y, v0)) 40 | 41 | attach_locally(globals) 42 | y %<-% { 43 | x <- 1:10 44 | sumtwo(a + b * x) 45 | } %globals% TRUE 46 | rm(list = names(globals)) 47 | print(y) 48 | stopifnot(all.equal(y, v0)) 49 | 50 | ## No need to search for globals 51 | y %<-% { 1 } %globals% FALSE 52 | print(y) 53 | stopifnot(identical(y, 1)) 54 | 55 | ## Exception - missing global 56 | attach_locally(globals) 57 | f <- future({ 58 | x <- 1:10 59 | sumtwo(a + b * x) 60 | }, globals = FALSE) 61 | rm(list = names(globals)) 62 | y <- tryCatch(value(f), error = identity) 63 | if (!inherits(f, c("SequentialFuture", "MulticoreFuture"))) { 64 | stopifnot(inherits(y, "simpleError")) 65 | } 66 | 67 | message("*** Globals - automatic ... DONE") 68 | 69 | 70 | message("*** Globals manually specified as named list ...") 71 | 72 | ## Make sure globals do not exist 73 | rm(list = names(globals)) 74 | 75 | f <- future({ 76 | x <- 1:10 77 | sumtwo(a + b * x) 78 | }, globals = globals) 79 | v <- value(f) 80 | print(v) 81 | stopifnot(all.equal(v, v0)) 82 | 83 | y %<-% { 84 | x <- 1:10 85 | sumtwo(a + b * x) 86 | } %globals% globals 87 | print(y) 88 | stopifnot(all.equal(y, v0)) 89 | 90 | message("*** Globals manually specified as named list ... DONE") 91 | 92 | 93 | message("*** Globals manually specified by their names ...") 94 | 95 | attach_locally(globals) 96 | f <- future({ 97 | x <- 1:10 98 | sumtwo(a + b * x) 99 | }, globals = c("a", "b", "sumtwo")) 100 | rm(list = names(globals)) 101 | v <- value(f) 102 | print(v) 103 | stopifnot(all.equal(v, v0)) 104 | 105 | attach_locally(globals) 106 | y %<-% { 107 | x <- 1:10 108 | sumtwo(a + b * x) 109 | } %globals% c("a", "b", "sumtwo") 110 | rm(list = names(globals)) 111 | print(y) 112 | stopifnot(all.equal(y, v0)) 113 | 114 | message("*** Globals manually specified by their names ... DONE") 115 | 116 | 117 | message("*** Globals - manually ... DONE") 118 | 119 | source("incl/end.R") 120 | -------------------------------------------------------------------------------- /tests/globals,subassignment.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | plan(batchtools_local) 4 | 5 | ## CRAN processing times: 6 | ## On Windows 32-bit, don't run these tests on batchtools 7 | if (!fullTest && isWin32) plan(sequential) 8 | 9 | oopts <- c(oopts, options( 10 | future.globals.resolve = TRUE, 11 | future.globals.onMissing = "error" 12 | )) 13 | 14 | 15 | message("*** Globals - subassignments ...") 16 | 17 | message("*** Globals - subassignments w/ x$a <- value ...") 18 | 19 | ## Truth: 20 | x <- x0 <- list() 21 | y0 <- list(a = 1) 22 | str(list(x = x, y0 = y0)) 23 | 24 | y <- local({ 25 | x$a <- 1 26 | x 27 | }) 28 | stopifnot(identical(y, y0)) 29 | 30 | y <- local({ 31 | x[["a"]] <- 1 32 | x 33 | }) 34 | stopifnot(identical(y, y0)) 35 | 36 | y <- local({ 37 | x["a"] <- list(1) 38 | x 39 | }) 40 | stopifnot(identical(y, y0)) 41 | 42 | stopifnot(identical(x, list())) 43 | 44 | ## Explicit future 45 | x <- list() 46 | f <- future({ 47 | x$a <- 1 48 | x 49 | }) 50 | rm(list = "x") 51 | y <- value(f) 52 | print(y) 53 | stopifnot(identical(y, y0)) 54 | 55 | ## Future assignment 56 | x <- list() 57 | y %<-% { 58 | x$a <- 1 59 | x 60 | } 61 | rm(list = "x") 62 | print(y) 63 | stopifnot(identical(y, y0)) 64 | 65 | ## 'x' is _not_ a global variable here 66 | x <- list() 67 | y %<-% { 68 | x <- list(b = 2) 69 | x$a <- 1 70 | x 71 | } 72 | rm(list = "x") 73 | print(y) 74 | stopifnot(identical(y, list(b = 2, a = 1))) 75 | 76 | ## Explicit future 77 | x <- list() 78 | f <- future({ 79 | x[["a"]] <- 1 80 | x 81 | }) 82 | rm(list = "x") 83 | y <- value(f) 84 | print(y) 85 | stopifnot(identical(y, y0)) 86 | 87 | ## Future assignment 88 | x <- list() 89 | y %<-% { 90 | x[["a"]] <- 1 91 | x 92 | } 93 | rm(list = "x") 94 | print(y) 95 | stopifnot(identical(y, y0)) 96 | 97 | ## Explicit future 98 | x <- list() 99 | f <- future({ 100 | x["a"] <- list(1) 101 | x 102 | }) 103 | rm(list = "x") 104 | y <- value(f) 105 | print(y) 106 | stopifnot(identical(y, y0)) 107 | 108 | ## Future assignment 109 | x <- list() 110 | y %<-% { 111 | x["a"] <- list(1) 112 | x 113 | } 114 | rm(list = "x") 115 | print(y) 116 | stopifnot(identical(y, y0)) 117 | 118 | ## Future assignment 119 | x <- list() 120 | name <- "a" 121 | y %<-% { 122 | x[name] <- list(1) 123 | x 124 | } 125 | rm(list = c("x", "name")) 126 | print(y) 127 | stopifnot(identical(y, y0)) 128 | 129 | message("*** Globals - subassignments w/ x$a <- value ... DONE") 130 | 131 | message("*** Globals - subassignments ... DONE") 132 | 133 | source("incl/end.R") 134 | -------------------------------------------------------------------------------- /tests/globals,tricky.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | library("listenv") 3 | 4 | plan(batchtools_local) 5 | 6 | ## CRAN processing times: 7 | ## On Windows 32-bit, don't run these tests on batchtools 8 | if (!fullTest && isWin32) plan(sequential) 9 | 10 | message("*** Tricky use cases related to globals ...") 11 | 12 | message("- Globals with the same name as 'base' objects ...") 13 | 14 | ## 'col' is masked by 'base::col' (Issue #55) 15 | col <- 3 16 | x %<-% { stopifnot(is.numeric(col)); col } 17 | print(x) 18 | stopifnot(x == col) 19 | 20 | ## https://github.com/mllg/batchtools/issues/88 21 | message("- Globals that don't necessarily map to filenames ...") 22 | .a <- 42L 23 | x %<-% { .a } 24 | print(x) 25 | stopifnot(x == .a) 26 | 27 | `$foo` <- 42L 28 | x %<-% { `$foo` } 29 | print(x) 30 | stopifnot(x == `$foo`) 31 | 32 | 33 | message("- flapply(x, FUN = base::vector, ...) ...") 34 | 35 | flapply <- function(x, FUN, ...) { 36 | res <- listenv() 37 | for (ii in seq_along(x)) { 38 | res[[ii]] %<-% FUN(x[[ii]], ...) 39 | } 40 | names(res) <- names(x) 41 | 42 | ## Make sure 'x', 'FUN' and 'ii' are truly 43 | ## exported to the future environment 44 | rm(list = c("x", "FUN", "ii")) 45 | 46 | as.list(res) 47 | } 48 | 49 | x <- list(a = "integer", c = "character", c = "list") 50 | str(list(x = x)) 51 | 52 | y0 <- lapply(x, FUN = base::vector, length = 2L) 53 | str(list(y0 = y0)) 54 | 55 | y <- flapply(x, FUN = base::vector, length = 2L) 56 | str(list(y = y)) 57 | stopifnot(identical(y, y0)) 58 | 59 | 60 | message("- flapply(x, FUN = future:::hpaste, ...) ...") 61 | 62 | x <- list(a = c("hello", b = 1:100)) 63 | str(list(x = x)) 64 | 65 | y0 <- lapply(x, FUN = future:::hpaste, collapse = "; ", maxHead = 3L) 66 | str(list(y0 = y0)) 67 | 68 | y <- flapply(x, FUN = future:::hpaste, collapse = "; ", maxHead = 3L) 69 | str(list(y = y)) 70 | stopifnot(identical(y, y0)) 71 | 72 | 73 | message("- flapply(x, FUN = listenv::listenv, ...) ...") 74 | 75 | x <- list() 76 | 77 | y <- listenv() 78 | y$A <- 3L 79 | x$a <- y 80 | 81 | y <- listenv() 82 | y$A <- 3L 83 | y$B <- c("hello", b = 1:100) 84 | x$b <- y 85 | 86 | print(x) 87 | 88 | y0 <- lapply(x, FUN = listenv::mapping) 89 | str(list(y0 = y0)) 90 | 91 | y <- flapply(x, FUN = listenv::mapping) 92 | str(list(y = y)) 93 | stopifnot(identical(y, y0)) 94 | 95 | 96 | message("*** Tricky use cases related to globals ... DONE") 97 | 98 | source("incl/end.R") 99 | -------------------------------------------------------------------------------- /tests/incl/end.R: -------------------------------------------------------------------------------- 1 | ## Undo future strategy 2 | future::plan(oplan) 3 | 4 | 5 | ## Undo options 6 | ## (a) Added 7 | added <- setdiff(names(options()), names(oopts0)) 8 | opts <- vector("list", length = length(added)) 9 | names(opts) <- added 10 | options(opts) 11 | ## (b) Modified 12 | options(oopts) 13 | ## (c) Assert that everything was undone 14 | stopifnot(all.equal(options(), oopts0)) 15 | 16 | 17 | ## Undo system environment variables 18 | ## (a) Added 19 | cenvs <- Sys.getenv() 20 | added <- setdiff(names(cenvs), names(oenvs0)) 21 | for (name in added) Sys.unsetenv(name) 22 | ## (b) Missing 23 | missing <- setdiff(names(oenvs0), names(cenvs)) 24 | if (length(missing) > 0) { 25 | values <- oenvs0[missing] 26 | do.call(Sys.setenv, as.list(values)) 27 | ## WORKAROUND: Most platforms allow setting an environment variable to 28 | ## "", but Windows does not and there Sys.setenv(FOO = "") unsets FOO. 29 | if (.Platform$OS.type == "windows") { 30 | drop <- missing[!nzchar(values)] 31 | if (length(drop) > 0) { 32 | oenvs0 <- oenvs0[setdiff(names(oenvs0), drop)] 33 | ## In case Sys.setenv() supports empty string in the future 34 | Sys.unsetenv(drop) 35 | } 36 | } 37 | } 38 | ## (c) Modified? 39 | for (name in intersect(names(cenvs), names(oenvs0))) { 40 | ## WORKAROUND: On Linux Wine, base::Sys.getenv() may 41 | ## return elements with empty names. /HB 2016-10-06 42 | if (nchar(name) == 0) next 43 | if (!identical(cenvs[[name]], oenvs0[[name]])) { 44 | do.call(Sys.setenv, as.list(oenvs0[name])) 45 | ## WORKAROUND: Most platforms allow setting an environment variable to 46 | ## "", but Windows does not and there Sys.setenv(FOO = "") unsets FOO. 47 | if (.Platform$OS.type == "windows" && !nzchar(oenvs0[[name]])) { 48 | oenvs0 <- oenvs0[setdiff(names(oenvs0), name)] 49 | ## In case Sys.setenv() supports empty string in the future 50 | Sys.unsetenv(name) 51 | } 52 | } 53 | } 54 | ## (d) Assert that everything was undone 55 | if (!identical(Sys.getenv(), oenvs0)) { 56 | diff <- setdiff(names(Sys.getenv()), names(oenvs0)) 57 | message(sprintf("Env vars added: [n=%d] %s", length(diff), paste(sQuote(diff), collapse = ", "))) 58 | diff <- setdiff(names(oenvs0), names(Sys.getenv())) 59 | message(sprintf("Env vars removed: [n=%d] %s", length(diff), paste(sQuote(diff), collapse = ", "))) 60 | } 61 | stopifnot(all.equal(Sys.getenv(), oenvs0)) 62 | 63 | 64 | ## Undo variables 65 | rm(list = c(setdiff(ls(), ovars))) 66 | 67 | 68 | ## Travis CI specific: Explicit garbage collection because it 69 | ## looks like Travis CI might run out of memory during 'covr' 70 | ## testing and we now have so many tests. /HB 2017-01-11 71 | if ("covr" %in% loadedNamespaces()) gc() 72 | -------------------------------------------------------------------------------- /tests/incl/start,load-only.R: -------------------------------------------------------------------------------- 1 | ## Record original state 2 | ovars <- ls() 3 | oenvs <- oenvs0 <- Sys.getenv() 4 | oopts0 <- options() 5 | 6 | covr_testing <- ("covr" %in% loadedNamespaces()) 7 | on_solaris <- grepl("^solaris", R.version$os) 8 | on_macos <- grepl("^darwin", R.version$os) 9 | on_githubactions <- as.logical(Sys.getenv("GITHUB_ACTIONS", "FALSE")) 10 | 11 | ## Default options 12 | oopts <- options( 13 | warn = 1L, 14 | mc.cores = 2L, 15 | future.debug = FALSE, 16 | future.wait.interval = 0.1, ## Speed up await() and delete() 17 | ## Reset the following during testing in case 18 | ## they are set on the test system 19 | future.availableCores.system = NULL, 20 | future.availableCores.fallback = NULL 21 | ) 22 | oopts$future.delete <- getOption("future.delete") 23 | oplan <- future::plan() 24 | 25 | ## In case it set outside, reset: 26 | options(future.batchtools.workers = NULL) 27 | Sys.unsetenv("R_FUTURE_BATCHTOOLS_WORKERS") 28 | 29 | path <- Sys.getenv("R_BATCHTOOLS_SEARCH_PATH") 30 | if (!nzchar(path)) { 31 | path <- system.file(package = "future.batchtools", 32 | "templates-for-R_CMD_check", mustWork = TRUE) 33 | Sys.setenv(R_BATCHTOOLS_SEARCH_PATH = path) 34 | } else { 35 | warning("Using a non-standard R_BATCHTOOLS_SEARCH_PATH while testing: ", 36 | sQuote(path)) 37 | if (!file_test("-d", path)) { 38 | stop("R_BATCHTOOLS_SEARCH_PATH specifies a non-existing folder: ", 39 | sQuote(path)) 40 | } 41 | } 42 | 43 | ## Use local batchtools futures by default 44 | future::plan(future.batchtools::batchtools_local) 45 | 46 | fullTest <- (Sys.getenv("_R_CHECK_FULL_") != "") 47 | 48 | isWin32 <- (.Platform$OS.type == "windows" && .Platform$r_arch == "i386") 49 | 50 | all_strategies <- local({ 51 | .cache <- NULL 52 | function(envir = parent.frame()) { 53 | if (!is.null(.cache)) return(.cache) 54 | 55 | strategies <- Sys.getenv("R_FUTURE_TESTS_STRATEGIES") 56 | strategies <- unlist(strsplit(strategies, split = ",")) 57 | strategies <- gsub(" ", "", strategies) 58 | strategies <- strategies[nzchar(strategies)] 59 | 60 | ## When testing for instance 'batchtools_sge', look for a customize 61 | ## template file, e.g. R_BATCHTOOLS_SEARCH_PATH/batchtools.sge.tmpl 62 | if (length(strategies) > 0L) { 63 | ## If there is a custom R_BATCHTOOLS_SEARCH_PATH/setup.R' file, run it 64 | pathname <- file.path(path, "setup.R") 65 | if (file_test("-f", pathname)) source(pathname, local = envir) 66 | } 67 | 68 | strategies <- c(future:::supportedStrategies(), strategies) 69 | strategies <- unique(strategies) 70 | .cache <<- strategies 71 | 72 | strategies 73 | } 74 | }) 75 | 76 | test_strategy <- function(strategy) { 77 | strategy %in% all_strategies() 78 | } 79 | 80 | attached_packages <- future.batchtools:::attached_packages 81 | await <- future.batchtools:::await 82 | delete <- future.batchtools:::delete 83 | import_future <- future.batchtools:::import_future 84 | is_false <- future.batchtools:::is_false 85 | is_na <- future.batchtools:::is_na 86 | is_os <- future.batchtools:::is_os 87 | hpaste <- future.batchtools:::hpaste 88 | mcat <- function(...) message(..., appendLF = FALSE) 89 | mprintf <- function(...) message(sprintf(...), appendLF = FALSE) 90 | mprint <- future.batchtools:::mprint 91 | mstr <- future.batchtools:::mstr 92 | printf <- future.batchtools:::printf 93 | temp_registry <- future.batchtools:::temp_registry 94 | trim <- future.batchtools:::trim 95 | attach_locally <- function(x, envir = parent.frame()) { 96 | for (name in names(x)) { 97 | assign(name, value = x[[name]], envir = envir) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /tests/incl/start.R: -------------------------------------------------------------------------------- 1 | library("future.batchtools") 2 | source("incl/start,load-only.R") 3 | -------------------------------------------------------------------------------- /tests/nbrOfWorkers.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | library("listenv") 3 | 4 | message("*** nbrOfWorkers() ...") 5 | 6 | message("*** nbrOfWorkers() - local, interactive ...") 7 | 8 | n <- nbrOfWorkers(batchtools_local) 9 | message("Number of workers: ", n) 10 | stopifnot(n == 1L) 11 | 12 | n <- nbrOfFreeWorkers(batchtools_local) 13 | message("Number of free workers: ", n) 14 | stopifnot(n == 1L) 15 | 16 | n <- nbrOfFreeWorkers(batchtools_local, background = TRUE) 17 | message("Number of free background workers: ", n) 18 | stopifnot(n == 0L) 19 | 20 | n <- nbrOfWorkers(batchtools_interactive) 21 | message("Number of workers: ", n) 22 | stopifnot(n == 1L) 23 | 24 | n <- nbrOfFreeWorkers(batchtools_interactive) 25 | message("Number of free workers: ", n) 26 | stopifnot(n == 1L) 27 | 28 | n <- nbrOfFreeWorkers(batchtools_interactive, background = TRUE) 29 | message("Number of free background workers: ", n) 30 | stopifnot(n == 0L) 31 | 32 | 33 | plan(batchtools_local) 34 | n <- nbrOfWorkers() 35 | message("Number of workers: ", n) 36 | stopifnot(n == 1L) 37 | 38 | n <- nbrOfFreeWorkers() 39 | message("Number of free workers: ", n) 40 | stopifnot(n == 1L) 41 | 42 | n <- nbrOfFreeWorkers(background = TRUE) 43 | message("Number of free background workers: ", n) 44 | stopifnot(n == 0L) 45 | 46 | plan(batchtools_interactive) 47 | n <- nbrOfWorkers() 48 | message("Number of workers: ", n) 49 | stopifnot(n == 1L) 50 | 51 | n <- nbrOfFreeWorkers() 52 | message("Number of free workers: ", n) 53 | stopifnot(n == 1L) 54 | 55 | n <- nbrOfFreeWorkers(background = TRUE) 56 | message("Number of free background workers: ", n) 57 | stopifnot(n == 0L) 58 | 59 | 60 | message("*** nbrOfWorkers() - local, interactive ... DONE") 61 | 62 | ncores <- availableCores("multicore") 63 | if (ncores >= 2L) { 64 | message("*** nbrOfWorkers() - multicore ...") 65 | 66 | n <- nbrOfWorkers(batchtools_multicore) 67 | message("Number of workers: ", n) 68 | stopifnot(n == ncores) 69 | 70 | n <- nbrOfFreeWorkers(batchtools_multicore) 71 | message("Number of free workers: ", n) 72 | stopifnot(n == ncores) 73 | 74 | n <- nbrOfFreeWorkers(batchtools_multicore, background = TRUE) 75 | message("Number of free background workers: ", n) 76 | stopifnot(n == ncores) 77 | 78 | plan(batchtools_multicore, workers = 2L) 79 | n <- nbrOfWorkers() 80 | message("Number of workers: ", n) 81 | stopifnot(n == 2L) 82 | 83 | n <- nbrOfFreeWorkers() 84 | message("Number of free workers: ", n) 85 | stopifnot(n == 2L) 86 | 87 | n <- nbrOfFreeWorkers(background = TRUE) 88 | message("Number of free background workers: ", n) 89 | stopifnot(n == 2L) 90 | 91 | workers <- min(2L, ncores) 92 | plan(batchtools_multicore, workers = workers) 93 | n <- nbrOfWorkers() 94 | message("Number of workers: ", n) 95 | stopifnot(n == workers) 96 | 97 | message("*** nbrOfWorkers() - multicore ... DONE") 98 | } ## if (ncores >= 2L) 99 | 100 | 101 | message("*** nbrOfWorkers() - templates ...") 102 | 103 | ## Test with +Inf workers 104 | options(future.batchtools.workers = +Inf) 105 | 106 | n <- nbrOfWorkers(batchtools_lsf) 107 | message("Number of workers: ", n) 108 | stopifnot(is.infinite(n)) 109 | 110 | n <- nbrOfWorkers(batchtools_openlava) 111 | message("Number of workers: ", n) 112 | stopifnot(is.infinite(n)) 113 | 114 | n <- nbrOfWorkers(batchtools_sge) 115 | message("Number of workers: ", n) 116 | stopifnot(is.infinite(n)) 117 | 118 | n <- nbrOfWorkers(batchtools_slurm) 119 | message("Number of workers: ", n) 120 | stopifnot(is.infinite(n)) 121 | 122 | n <- nbrOfWorkers(batchtools_torque) 123 | message("Number of workers: ", n) 124 | stopifnot(is.infinite(n)) 125 | 126 | message("*** nbrOfWorkers() - templates ... DONE") 127 | 128 | message("*** nbrOfWorkers() - custom ...") 129 | 130 | cf <- batchtools::makeClusterFunctionsInteractive(external = TRUE) 131 | str(cf) 132 | 133 | plan(batchtools_custom, cluster.functions = cf) 134 | n <- nbrOfWorkers() 135 | message("Number of workers: ", n) 136 | stopifnot(n == 1L) 137 | 138 | message("*** nbrOfWorkers() - custom ... DONE") 139 | 140 | message("*** nbrOfWorkers() ... DONE") 141 | 142 | source("incl/end.R") 143 | -------------------------------------------------------------------------------- /tests/plan.R: -------------------------------------------------------------------------------- 1 | source("incl/start,load-only.R") 2 | 3 | message("*** plan() ...") 4 | 5 | message("*** future::plan(future.batchtools::batchtools_local)") 6 | oplan <- future::plan(future.batchtools::batchtools_local) 7 | print(future::plan()) 8 | future::plan(oplan) 9 | print(future::plan()) 10 | 11 | 12 | library("future.batchtools") 13 | 14 | for (type in c("batchtools_interactive", "batchtools_local")) { 15 | mprintf("*** plan('%s') ...\n", type) 16 | 17 | plan(type) 18 | stopifnot(inherits(plan("next"), "batchtools")) 19 | 20 | a <- 0 21 | f <- future({ 22 | b <- 3 23 | c <- 2 24 | a * b * c 25 | }) 26 | a <- 7 ## Make sure globals are frozen 27 | v <- value(f) 28 | print(v) 29 | stopifnot(v == 0) 30 | 31 | 32 | ## Customize the 'work.dir' of the batchtools registries 33 | normalize_path <- function(path) { 34 | if (!utils::file_test("-d", path)) stop("No such path: ", path) 35 | opwd <- getwd() 36 | on.exit(setwd(opwd)) 37 | setwd(normalizePath(path)) 38 | getwd() 39 | } 40 | plan(type, registry = list(work.dir = NULL)) 41 | f <- future(42, lazy = TRUE) 42 | ## In future releases, lazy futures may stay vanilla Future objects 43 | if (inherits(f, "BatchtoolsFuture")) { 44 | if (!is.null(f$config$reg)) { 45 | utils::str(list( 46 | normalize_path(f$config$reg$work.dir), 47 | getwd = getwd() 48 | )) 49 | stopifnot(normalize_path(f$config$reg$work.dir) == getwd()) 50 | } 51 | } 52 | 53 | path <- tempdir() 54 | plan(type, registry = list(work.dir = path)) 55 | f <- future(42, lazy = TRUE) 56 | ## In future releases, lazy futures may stay vanilla Future objects 57 | if (inherits(f, "BatchtoolsFuture")) { 58 | if (!is.null(f$config$reg)) { 59 | utils::str(list( 60 | normalize_path(f$config$reg$work.dir), 61 | path = normalize_path(path) 62 | )) 63 | stopifnot(normalize_path(f$config$reg$work.dir) == normalize_path(path)) 64 | } 65 | } 66 | 67 | mprintf("*** plan('%s') ... DONE\n", type) 68 | } # for (type ...) 69 | 70 | 71 | message("*** Assert that default backend can be overridden ...") 72 | 73 | mpid <- Sys.getpid() 74 | print(mpid) 75 | 76 | plan(batchtools_interactive) 77 | pid %<-% { Sys.getpid() } 78 | print(pid) 79 | stopifnot(pid == mpid) 80 | 81 | plan(batchtools_local) 82 | pid %<-% { Sys.getpid() } 83 | print(pid) 84 | stopifnot(pid != mpid) 85 | 86 | 87 | message("*** plan() ... DONE") 88 | 89 | source("incl/end.R") 90 | -------------------------------------------------------------------------------- /tests/resources_OP.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | message("*** %resources% ...") 4 | 5 | plan(batchtools_local) 6 | 7 | ## This will test `%resources%` but it'll be ignored (with a warning) 8 | ## by batchtools_local() 9 | y %<-% { 42 } %resources% list(memory = 16000) ## 16,000 MiB of memory 10 | 11 | message("*** %resources% ... DONE") 12 | 13 | source("incl/end.R") 14 | -------------------------------------------------------------------------------- /tests/rng.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | options(future.debug = FALSE) 4 | 5 | message("*** RNG ...") 6 | 7 | plan(batchtools_local) 8 | 9 | message("- run() does not update RNG state") 10 | 11 | f1 <- future(1, lazy = TRUE) 12 | f2 <- future(2, lazy = TRUE) 13 | 14 | rng0 <- globalenv()$.Random.seed 15 | 16 | f1 <- run(f1) 17 | stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? 18 | 19 | f2 <- run(f2) 20 | stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? 21 | 22 | v1 <- value(f1) 23 | stopifnot(identical(v1, 1)) 24 | 25 | v2 <- value(f2) 26 | stopifnot(identical(v2, 2)) 27 | 28 | 29 | message("- future() does not update RNG state") 30 | 31 | rng0 <- globalenv()$.Random.seed 32 | 33 | f1 <- future(1) 34 | stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? 35 | 36 | f2 <- future(2) 37 | stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? 38 | 39 | v1 <- value(f1) 40 | stopifnot(identical(v1, 1)) 41 | 42 | v2 <- value(f2) 43 | stopifnot(identical(v2, 2)) 44 | 45 | 46 | message("- resolved() does not update RNG state") 47 | 48 | f1 <- future(1) 49 | f2 <- future(2) 50 | 51 | rng0 <- globalenv()$.Random.seed 52 | 53 | d1 <- resolved(f1) 54 | stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? 55 | 56 | d2 <- resolved(f2) 57 | stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? 58 | 59 | v1 <- value(f1) 60 | stopifnot(identical(v1, 1)) 61 | 62 | v2 <- value(f2) 63 | stopifnot(identical(v2, 2)) 64 | 65 | 66 | message("- result() does not update RNG state") 67 | 68 | f1 <- future(1) 69 | f2 <- future(2) 70 | 71 | rng0 <- globalenv()$.Random.seed 72 | 73 | r1 <- result(f1) 74 | stopifnot(identical(r1$value, 1)) 75 | stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? 76 | 77 | r2 <- result(f2) 78 | stopifnot(identical(r2$value, 2)) 79 | stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? 80 | 81 | v1 <- value(f1) 82 | stopifnot(identical(v1, 1)) 83 | 84 | v2 <- value(f2) 85 | stopifnot(identical(v2, 2)) 86 | 87 | 88 | message("- value() does not update RNG state") 89 | 90 | f1 <- future(1) 91 | f2 <- future(2) 92 | 93 | rng0 <- globalenv()$.Random.seed 94 | 95 | v1 <- value(f1) 96 | stopifnot(identical(v1, 1)) 97 | stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? 98 | 99 | v2 <- value(f2) 100 | stopifnot(identical(v2, 2)) 101 | stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? 102 | 103 | message("*** RNG ... DONE") 104 | 105 | source("incl/end.R") 106 | -------------------------------------------------------------------------------- /tests/stdout.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | message("*** Standard output ...") 4 | 5 | truth_rows <- utils::capture.output({ 6 | print(1:20) 7 | str(1:20) 8 | cat(letters, sep = "-") 9 | cat(1:6, collapse = "\n") 10 | write.table(datasets::iris[1:10,], sep = "\t") 11 | }) 12 | truth <- paste0(paste(truth_rows, collapse = "\n"), "\n") 13 | print(truth) 14 | 15 | message("batchtools_local ...") 16 | plan(batchtools_local) 17 | 18 | for (stdout in c(TRUE, FALSE, NA)) { 19 | message(sprintf("- stdout = %s", stdout)) 20 | 21 | f <- future({ 22 | print(1:20) 23 | str(1:20) 24 | cat(letters, sep = "-") 25 | cat(1:6, collapse = "\n") 26 | write.table(datasets::iris[1:10,], sep = "\t") 27 | 42L 28 | }, stdout = stdout) 29 | r <- result(f) 30 | str(r) 31 | stopifnot(value(f) == 42L) 32 | 33 | if (is.na(stdout)) { 34 | stopifnot(is.null(r$stdout) || r$stdout == "") 35 | } else if (stdout) { 36 | print(r) 37 | stopifnot(identical(r$stdout, truth)) 38 | } else { 39 | stopifnot(is.null(r$stdout)) 40 | } 41 | 42 | v %<-% { 43 | print(1:20) 44 | str(1:20) 45 | cat(letters, sep = "-") 46 | cat(1:6, collapse = "\n") 47 | write.table(datasets::iris[1:10,], sep = "\t") 48 | 42L 49 | } %stdout% stdout 50 | out <- utils::capture.output(y <- v) 51 | stopifnot(y == 42L) 52 | 53 | if (is.na(stdout) || !stdout) { 54 | stopifnot(out == "") 55 | } else { 56 | print(out) 57 | stopifnot(identical(out, truth_rows)) 58 | } 59 | } ## for (stdout ...) 60 | 61 | message("batchtools_local ... DONE") 62 | 63 | message("*** Standard output ... DONE") 64 | 65 | source("incl/end.R") 66 | -------------------------------------------------------------------------------- /tests/utils.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | message("*** Utility functions ...") 4 | 5 | message("- is_na() ...") 6 | stopifnot(is_na(NA), !is_na(TRUE), !is_na(FALSE), !is_na(1), 7 | !is_na(NULL), !is_na(1:2), !is_na(rep(NA, times = 3)), 8 | !is_na(rep(TRUE, 3)), !is_na(letters)) 9 | 10 | message("- is_false() ...") 11 | stopifnot(is_false(FALSE), !is_false(TRUE), !is_false(NA), !is_false(1), 12 | !is_false(NULL), !is_false(1:2), !is_false(rep(FALSE, times = 3)), 13 | !is_false(rep(TRUE, times = 3)), !is_false(letters)) 14 | 15 | message("- attached_packages() ...") 16 | print(attached_packages()) 17 | 18 | 19 | message("- hpaste() & printf() ...") 20 | # Some vectors 21 | x <- 1:6 22 | y <- 10:1 23 | z <- LETTERS[x] 24 | 25 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 26 | # Abbreviation of output vector 27 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 28 | printf("x = %s.\n", hpaste(x)) 29 | ## x = 1, 2, 3, ..., 6. 30 | 31 | printf("x = %s.\n", hpaste(x, max_head = 2)) 32 | ## x = 1, 2, ..., 6. 33 | 34 | printf("x = %s.\n", hpaste(x, max_head = 3)) # Default 35 | ## x = 1, 2, 3, ..., 6. 36 | 37 | # It will never output 1, 2, 3, 4, ..., 6 38 | printf("x = %s.\n", hpaste(x, max_head = 4)) 39 | ## x = 1, 2, 3, 4, 5 and 6. 40 | 41 | # Showing the tail 42 | printf("x = %s.\n", hpaste(x, max_head = 1, max_tail = 2)) 43 | ## x = 1, ..., 5, 6. 44 | 45 | # Turning off abbreviation 46 | printf("y = %s.\n", hpaste(y, max_head = Inf)) 47 | ## y = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 48 | 49 | ## ...or simply 50 | printf("y = %s.\n", paste(y, collapse = ", ")) 51 | ## y = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 52 | 53 | # Adding a special separator before the last element 54 | # Change last separator 55 | printf("x = %s.\n", hpaste(x, last_collapse = " and ")) 56 | ## x = 1, 2, 3, 4, 5 and 6. 57 | 58 | message("- mcat(), mprintf(), mprint() and mstr() ...") 59 | mcat("Hello world!\n") 60 | mprintf("Hello %s!\n", "world") 61 | mprint("Hello world!") 62 | mstr("Hello world!") 63 | 64 | message("- trim() ...") 65 | mprint(trim(" hello ")) 66 | stopifnot(trim(" hello ") == "hello") 67 | 68 | 69 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 70 | # is_os() 71 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 72 | message("- is_os() ...") 73 | for (os in c("darwin", "freebsd", "irix", "linux", "openbsd", 74 | "solaris", "windows")) { 75 | mprintf("is_os('%s') = %s", os, is_os(os)) 76 | } 77 | 78 | 79 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 80 | # import_future() 81 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 82 | message("*** import_future() ...") 83 | 84 | future <- import_future("future") 85 | stopifnot(identical(future, future::future)) 86 | 87 | future <- import_future("", default = future::future) 88 | stopifnot(identical(future, future::future)) 89 | 90 | res <- try(import_future(""), silent = TRUE) 91 | stopifnot(inherits(res, "try-error")) 92 | 93 | message("*** import_future() ... DONE") 94 | 95 | message("*** Utility functions ... DONE") 96 | 97 | source("incl/end.R") 98 | -------------------------------------------------------------------------------- /tests/zzz,future_lapply.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | if (requireNamespace("future.apply", quietly = TRUE)) { 4 | future_lapply <- future.apply::future_lapply 5 | library("listenv") 6 | 7 | ## Setup all strategies including custom once for testing on HPC environments 8 | print(all_strategies()) 9 | 10 | message("All HPC strategies:") 11 | strategies <- c("batchtools_lsf", "batchtools_openlava", "batchtools_sge", 12 | "batchtools_slurm", "batchtools_torque") 13 | mprint(strategies, debug = TRUE) 14 | 15 | message("Supported HPC strategies:") 16 | strategies <- strategies[sapply(strategies, FUN = test_strategy)] 17 | mprint(strategies, debug = TRUE) 18 | 19 | strategies <- c("batchtools_local", strategies) 20 | 21 | if (fullTest) { 22 | strategies <- c("batchtools_interactive", strategies) 23 | 24 | batchtools_custom_local <- function(expr, substitute = TRUE, 25 | cluster.functions = batchtools::makeClusterFunctionsInteractive(external = TRUE), ...) { 26 | if (substitute) expr <- substitute(expr) 27 | batchtools_custom(expr, substitute = FALSE, ..., 28 | cluster.functions = cluster.functions) 29 | } 30 | class(batchtools_custom_local) <- c("batchtools_custom_local", 31 | class(batchtools_custom)) 32 | strategies <- c("batchtools_custom_local", strategies) 33 | } 34 | 35 | ## CRAN processing times: Don't run these tests on Windows 32-bit 36 | if (!fullTest && isWin32) strategies <- character(0L) 37 | 38 | message("Strategies to test with:") 39 | mprint(strategies, debug = TRUE) 40 | 41 | 42 | message("*** future_lapply() ...") 43 | 44 | message("- future_lapply(x, FUN = vector, ...) ...") 45 | 46 | x <- list(a = "integer", c = "character", c = "list") 47 | str(list(x = x)) 48 | 49 | y0 <- lapply(x, FUN = vector, length = 2L) 50 | str(list(y0 = y0)) 51 | 52 | for (strategy in strategies) { 53 | mprintf("- plan('%s') ...\n", strategy) 54 | plan(strategy) 55 | mprint(plan, debug = TRUE) 56 | if (nbrOfWorkers() > 2) plan(strategy, workers = 2L) 57 | stopifnot(nbrOfWorkers() < Inf) 58 | 59 | for (scheduling in list(FALSE, TRUE)) { 60 | y <- future_lapply(x, FUN = vector, length = 2L, 61 | future.scheduling = scheduling) 62 | str(list(y = y)) 63 | stopifnot(identical(y, y0)) 64 | } 65 | } 66 | 67 | 68 | message("- future_lapply(x, FUN = base::vector, ...) ...") 69 | 70 | x <- list(a = "integer", c = "character", c = "list") 71 | str(list(x = x)) 72 | 73 | y0 <- lapply(x, FUN = base::vector, length = 2L) 74 | str(list(y0 = y0)) 75 | 76 | for (strategy in strategies) { 77 | mprintf("- plan('%s') ...\n", strategy) 78 | plan(strategy) 79 | mprint(plan, debug = TRUE) 80 | if (nbrOfWorkers() > 2) plan(strategy, workers = 2L) 81 | stopifnot(nbrOfWorkers() < Inf) 82 | 83 | for (scheduling in list(FALSE, TRUE)) { 84 | y <- future_lapply(x, FUN = base::vector, length = 2L, 85 | future.scheduling = scheduling) 86 | str(list(y = y)) 87 | stopifnot(identical(y, y0)) 88 | } 89 | } 90 | 91 | message("- future_lapply(x, FUN = future:::hpaste, ...) ...") 92 | 93 | x <- list(a = c("hello", b = 1:100)) 94 | str(list(x = x)) 95 | 96 | y0 <- lapply(x, FUN = future:::hpaste, collapse = "; ", maxHead = 3L) 97 | str(list(y0 = y0)) 98 | 99 | for (strategy in strategies) { 100 | mprintf("- plan('%s') ...\n", strategy) 101 | plan(strategy) 102 | mprint(plan, debug = TRUE) 103 | if (nbrOfWorkers() > 2) plan(strategy, workers = 2L) 104 | stopifnot(nbrOfWorkers() < Inf) 105 | 106 | for (scheduling in list(FALSE, TRUE)) { 107 | y <- future_lapply(x, FUN = future:::hpaste, collapse = "; ", 108 | maxHead = 3L, future.scheduling = scheduling) 109 | str(list(y = y)) 110 | stopifnot(identical(y, y0)) 111 | } 112 | } 113 | 114 | 115 | message("- future_lapply(x, FUN = listenv::listenv, ...) ...") 116 | 117 | x <- list() 118 | 119 | y <- listenv() 120 | y$A <- 3L 121 | x$a <- y 122 | 123 | y <- listenv() 124 | y$A <- 3L 125 | y$B <- c("hello", b = 1:100) 126 | x$b <- y 127 | 128 | print(x) 129 | 130 | y0 <- lapply(x, FUN = listenv::mapping) 131 | str(list(y0 = y0)) 132 | 133 | for (strategy in strategies) { 134 | mprintf("- plan('%s') ...\n", strategy) 135 | plan(strategy) 136 | if (nbrOfWorkers() > 2) plan(strategy, workers = 2L) 137 | stopifnot(nbrOfWorkers() < Inf) 138 | 139 | for (scheduling in list(FALSE, TRUE)) { 140 | y <- future_lapply(x, FUN = listenv::mapping, future.scheduling = scheduling) 141 | str(list(y = y)) 142 | stopifnot(identical(y, y0)) 143 | } 144 | } 145 | 146 | 147 | message("- future_lapply(x, FUN, ...) for large length(x) ...") 148 | a <- 3.14 149 | x <- 1:1e5 150 | 151 | y <- future_lapply(x, FUN = function(z) sqrt(z + a)) 152 | y <- unlist(y, use.names = FALSE) 153 | 154 | stopifnot(all.equal(y, sqrt(x + a))) 155 | 156 | 157 | message("- future_lapply() with global in non-attached package ...") 158 | library("tools") 159 | my_ext <- function(x) file_ext(x) 160 | y_truth <- lapply("abc.txt", FUN = my_ext) 161 | 162 | for (strategy in strategies) { 163 | plan(strategy) 164 | if (nbrOfWorkers() > 2) plan(strategy, workers = 2L) 165 | stopifnot(nbrOfWorkers() < Inf) 166 | y <- future_lapply("abc.txt", FUN = my_ext) 167 | stopifnot(identical(y, y_truth)) 168 | } 169 | 170 | message("*** future_lapply() ... DONE") 171 | } 172 | 173 | source("incl/end.R") 174 | -------------------------------------------------------------------------------- /tests/zzz.onUnload.R: -------------------------------------------------------------------------------- 1 | source("incl/start.R") 2 | 3 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4 | # Load and unload of package 5 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 6 | loadNamespace("future.batchtools") 7 | 8 | message("*** .onUnload() ...") 9 | 10 | libpath <- dirname(system.file(package = "future.batchtools")) 11 | future.batchtools:::.onUnload(libpath) 12 | 13 | message("*** .onUnload() ... DONE") 14 | 15 | source("incl/end.R") 16 | --------------------------------------------------------------------------------