├── README_cache ├── gfm │ ├── __packages │ ├── unnamed-chunk-3_b1cda95f9f613c971ef3a8d63e3f554a.rdb │ ├── unnamed-chunk-2_00d877b98a39f6f6e4c275108bea86c2.RData │ ├── unnamed-chunk-2_00d877b98a39f6f6e4c275108bea86c2.rdb │ ├── unnamed-chunk-2_00d877b98a39f6f6e4c275108bea86c2.rdx │ ├── unnamed-chunk-3_b1cda95f9f613c971ef3a8d63e3f554a.RData │ ├── unnamed-chunk-3_b1cda95f9f613c971ef3a8d63e3f554a.cast │ └── unnamed-chunk-3_b1cda95f9f613c971ef3a8d63e3f554a.rdx └── html │ ├── __packages │ ├── unnamed-chunk-3_73fea0572b92743b58ff926a3b43baab.rdb │ ├── unnamed-chunk-2_91f857f630bd66849eb4330628798903.rdb │ ├── unnamed-chunk-2_91f857f630bd66849eb4330628798903.rdx │ ├── unnamed-chunk-3_73fea0572b92743b58ff926a3b43baab.cast │ ├── unnamed-chunk-3_73fea0572b92743b58ff926a3b43baab.rdx │ ├── unnamed-chunk-2_91f857f630bd66849eb4330628798903.RData │ └── unnamed-chunk-3_73fea0572b92743b58ff926a3b43baab.RData ├── .github ├── .gitignore └── workflows │ ├── pkgdown.yaml │ ├── R-CMD-check.yaml │ └── test-coverage.yaml ├── LICENSE ├── vignettes ├── .gitignore ├── images │ ├── maestro-publish.png │ └── maestro-posit-scheduled.png ├── maestro-4-directed-acyclic-graphs.Rmd └── maestro-1-quick-start.Rmd ├── data └── maestro_tags.rda ├── man ├── figures │ ├── logo.png │ ├── README-pressure-1.png │ └── README- │ │ ├── schedule-dark.svg │ │ ├── schedule.svg │ │ └── unnamed-chunk-3.svg ├── is_valid_dag.Rd ├── parse_rounding_unit.Rd ├── convert_to_seconds.Rd ├── last_run_errors.Rd ├── last_build_errors.Rd ├── maestro_parse_cli.Rd ├── build_schedule_entry.Rd ├── last_run_messages.Rd ├── last_run_warnings.Rd ├── get_flags.Rd ├── create_orchestrator.Rd ├── maestro.Rd ├── create_maestro.Rd ├── get_artifacts.Rd ├── get_status.Rd ├── get_schedule.Rd ├── show_network.Rd ├── suggest_orch_frequency.Rd ├── build_schedule.Rd ├── get_pipeline_run_sequence.Rd ├── invoke.Rd ├── get_slot_usage.Rd ├── create_pipeline.Rd ├── run_schedule.Rd └── MaestroSchedule.Rd ├── renv ├── .gitignore └── settings.json ├── tests ├── testthat │ ├── .DS_Store │ ├── test_pipelines_parse_all_bad │ │ ├── test_pipeline_script_with_error.R │ │ ├── dags_self_reference_outputs.R │ │ ├── dags_self_reference_inputs.R │ │ ├── tagged_but_no_func.R │ │ ├── dags.R │ │ ├── test_pipeline_daily_bad.R │ │ ├── test_pipeline_tz_bad.R │ │ ├── test_pipeline_no_func.R │ │ ├── tagged_but_no_func_multi.R │ │ └── specifiers.R │ ├── test_pipelines │ │ ├── test_pipeline_maestro.R │ │ ├── test_pipeline_inputs_bad.R │ │ ├── test_pipeline_outputs_bad.R │ │ ├── test_pipeline_inputs_good.R │ │ ├── test_pipeline_outputs_good.R │ │ ├── test_pipeline_hours_bad.R │ │ ├── test_pipeline_days_bad.R │ │ ├── test_pipeline_days_bad2.R │ │ ├── test_pipeline_days_bad3.R │ │ ├── test_pipeline_days_good.R │ │ ├── test_pipeline_hours_bad2.R │ │ ├── test_pipeline_hours_good.R │ │ ├── test_pipeline_days_good2.R │ │ ├── test_pipeline_months_bad.R │ │ ├── test_pipeline_months_good.R │ │ ├── test_pipeline_months_bad2.R │ │ ├── test_pipeline_daily_bad.R │ │ ├── test_pipeline_start_time_date.R │ │ ├── test_pipeline_start_time_int.R │ │ ├── test_pipeline_daily_default.R │ │ ├── test_pipeline_tz_bad.R │ │ ├── test_pipeline_daily_single_bad.R │ │ ├── test_pipeline_skip.R │ │ ├── test_pipeline_skip_bad.R │ │ ├── test_pipeline_loglevel_bad.R │ │ ├── test_pipeline_loglevel_good.R │ │ ├── test_pipeline_daily_good.R │ │ ├── test_pipeline_daily_single_good.R │ │ ├── test_multi_fun_pipeline.R │ │ └── test_multi_fun_pipeline_one_bad.R │ ├── test_pipelines_dags_bad │ │ └── dags.R │ ├── test_pipelines_dup_names │ │ └── dup_names.R │ ├── _snaps │ │ ├── MaestroSchedule.md │ │ ├── MaestroPipelineList.md │ │ ├── invoke.md │ │ ├── get_flags.md │ │ ├── get_schedule.md │ │ ├── dags.md │ │ ├── conditionals.md │ │ ├── MaestroPipeline.md │ │ ├── run_schedule.md │ │ └── get_slot_usage.md │ ├── test_pipelines_parse_all_good │ │ ├── pipe_with_custom_fun.R │ │ ├── dags.R │ │ ├── pipe2.R │ │ ├── specifiers.R │ │ └── pipe1.R │ ├── test_pipelines_run_all_good │ │ ├── chatty.R │ │ ├── pipe2.R │ │ └── pipe1.R │ ├── test_pipelines_parse_some_good │ │ ├── test_pipeline_daily_bad.R │ │ └── test_pipeline_daily_good.R │ ├── test_pipelines_run_artifacts │ │ └── artifacts.R │ ├── test_pipelines_run_two_warnings │ │ ├── pipe_warning.R │ │ ├── pipe2.R │ │ └── pipe1.R │ ├── test_pipelines_dags_run_bad │ │ └── dag_with_error.R │ ├── test_pipelines_run_logs_warn │ │ └── pipe_warning.R │ ├── test_pipelines_run_args_good │ │ └── pipe1.R │ ├── test_pipelines_run_some_errors │ │ ├── pipe2.R │ │ └── pipe1.R │ ├── test-get_status.R │ ├── test-get_artifacts.R │ ├── test-show_network.R │ ├── test-create_orchestrator.R │ ├── test-get_schedule.R │ ├── test_pipelines_run_specifiers │ │ └── specifiers.R │ ├── test_pipelines_run_skip │ │ └── with_skip.R │ ├── test-suggest_orch_frequency.R │ ├── test-create_maestro.R │ ├── test-utils.R │ ├── test-get_flags.R │ ├── test_pipelines_dags_good │ │ └── dags.R │ ├── test-build_schedule.R │ ├── test-MaestroSchedule.R │ ├── test-create_pipeline.R │ ├── test-MaestroPipeline.R │ ├── test-get_slot_usage.R │ ├── test-MaestroPipelineList.R │ ├── test-invoke.R │ ├── test-build_schedule_entry.R │ └── test-conditionals.R └── testthat.R ├── pkgdown ├── favicon │ ├── favicon.ico │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ ├── apple-touch-icon.png │ ├── apple-touch-icon-60x60.png │ ├── apple-touch-icon-76x76.png │ ├── apple-touch-icon-120x120.png │ ├── apple-touch-icon-152x152.png │ └── apple-touch-icon-180x180.png └── _pkgdown.yml ├── .dockerignore ├── CRAN-SUBMISSION ├── cran-comments.md ├── inst ├── orchestrator_template ├── rstudio │ └── templates │ │ └── project │ │ ├── maestro.png │ │ └── create_orchestrator.dcf ├── orchestrator_template_qmd ├── pipeline_template ├── sample_pipeline_dir │ ├── pipe2.R │ └── pipe1.R └── roxygen2-tags.yml ├── R ├── zzz.R ├── maestro.R ├── get_flags.R ├── get_artifacts.R ├── utils_cli.R ├── get_status.R ├── get_schedule.R ├── show_network.R ├── create_maestro.R ├── latest_getters.R ├── create_orchestrator.R ├── suggest_orch_frequency.R ├── build_schedule.R ├── get_slot_usage.R ├── invoke.R └── create_pipeline.R ├── pipelines ├── pipe2.R ├── pipe1.R └── my_etl.R ├── .gitignore ├── data-raw └── maestro_tags.R ├── codecov.yml ├── .Rbuildignore ├── maestro.Rproj ├── LICENSE.md ├── DESCRIPTION ├── NAMESPACE ├── README.md └── README.qmd /README_cache/gfm/__packages: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /README_cache/html/__packages: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | YEAR: 2024 2 | COPYRIGHT HOLDER: maestro authors 3 | -------------------------------------------------------------------------------- /README_cache/gfm/unnamed-chunk-3_b1cda95f9f613c971ef3a8d63e3f554a.rdb: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /README_cache/html/unnamed-chunk-3_73fea0572b92743b58ff926a3b43baab.rdb: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vignettes/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | *.R 3 | 4 | /.quarto/ 5 | 6 | **/*.quarto_ipynb 7 | -------------------------------------------------------------------------------- /data/maestro_tags.rda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/data/maestro_tags.rda -------------------------------------------------------------------------------- /man/figures/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/man/figures/logo.png -------------------------------------------------------------------------------- /renv/.gitignore: -------------------------------------------------------------------------------- 1 | library/ 2 | local/ 3 | cellar/ 4 | lock/ 5 | python/ 6 | sandbox/ 7 | staging/ 8 | -------------------------------------------------------------------------------- /tests/testthat/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/tests/testthat/.DS_Store -------------------------------------------------------------------------------- /pkgdown/favicon/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/pkgdown/favicon/favicon.ico -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_bad/test_pipeline_script_with_error.R: -------------------------------------------------------------------------------- 1 | stop("Here's a problem") 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .RData 2 | .Rhistory 3 | .git 4 | .gitignore 5 | manifest.json 6 | rsconnect/ 7 | .Rproj.user 8 | -------------------------------------------------------------------------------- /CRAN-SUBMISSION: -------------------------------------------------------------------------------- 1 | Version: 0.7.0 2 | Date: 2025-10-31 12:02:50 UTC 3 | SHA: efac2bb111d9a7b106d4276569769e47a243b960 4 | -------------------------------------------------------------------------------- /man/figures/README-pressure-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/man/figures/README-pressure-1.png -------------------------------------------------------------------------------- /pkgdown/favicon/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/pkgdown/favicon/favicon-16x16.png -------------------------------------------------------------------------------- /pkgdown/favicon/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/pkgdown/favicon/favicon-32x32.png -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_maestro.R: -------------------------------------------------------------------------------- 1 | #' @maestro 2 | i_like_defaults <- function() { 3 | 4 | } 5 | -------------------------------------------------------------------------------- /cran-comments.md: -------------------------------------------------------------------------------- 1 | ## Submission of 0.6.3 2 | 3 | 0 errors ✔ | 0 warnings ✔ | 0 notes ✔ 4 | 5 | Fixed test failures 6 | -------------------------------------------------------------------------------- /pkgdown/favicon/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/pkgdown/favicon/apple-touch-icon.png -------------------------------------------------------------------------------- /vignettes/images/maestro-publish.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/vignettes/images/maestro-publish.png -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_inputs_bad.R: -------------------------------------------------------------------------------- 1 | #' @maestroInputs 2 | some_inputs <- function(.input) { 3 | 4 | } 5 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_outputs_bad.R: -------------------------------------------------------------------------------- 1 | #' @maestroOutputs 2 | some_outputs <- function() { 3 | 4 | } 5 | -------------------------------------------------------------------------------- /inst/orchestrator_template: -------------------------------------------------------------------------------- 1 | library(maestro) 2 | 3 | schedule <- build_schedule() 4 | 5 | run_schedule( 6 | schedule 7 | ) 8 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_dags_bad/dags.R: -------------------------------------------------------------------------------- 1 | #' @maestroInputs im_not_here 2 | missing_inputs <- function(.input) { 3 | 4 | } 5 | -------------------------------------------------------------------------------- /inst/rstudio/templates/project/maestro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/inst/rstudio/templates/project/maestro.png -------------------------------------------------------------------------------- /pkgdown/favicon/apple-touch-icon-60x60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/pkgdown/favicon/apple-touch-icon-60x60.png -------------------------------------------------------------------------------- /pkgdown/favicon/apple-touch-icon-76x76.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/pkgdown/favicon/apple-touch-icon-76x76.png -------------------------------------------------------------------------------- /pkgdown/favicon/apple-touch-icon-120x120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/pkgdown/favicon/apple-touch-icon-120x120.png -------------------------------------------------------------------------------- /pkgdown/favicon/apple-touch-icon-152x152.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/pkgdown/favicon/apple-touch-icon-152x152.png -------------------------------------------------------------------------------- /pkgdown/favicon/apple-touch-icon-180x180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/pkgdown/favicon/apple-touch-icon-180x180.png -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_inputs_good.R: -------------------------------------------------------------------------------- 1 | #' @maestroInputs fun1 fun2 fun3 2 | some_inputs <- function(.input) { 3 | 4 | } 5 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_outputs_good.R: -------------------------------------------------------------------------------- 1 | #' @maestroOutputs fun1 fun2 fun3 2 | some_outputs <- function() { 3 | 4 | } 5 | -------------------------------------------------------------------------------- /vignettes/images/maestro-posit-scheduled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/vignettes/images/maestro-posit-scheduled.png -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_bad/dags_self_reference_outputs.R: -------------------------------------------------------------------------------- 1 | #' @maestroOutputs with_outputs 2 | with_outputs <- function() { 3 | 4 | } 5 | -------------------------------------------------------------------------------- /inst/orchestrator_template_qmd: -------------------------------------------------------------------------------- 1 | ```{r} 2 | library(maestro) 3 | 4 | schedule <- build_schedule() 5 | 6 | run_schedule( 7 | schedule 8 | ) 9 | ``` 10 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_hours_bad.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency hourly 2 | #' @maestroHours a 3 | one_hour <- function() { 4 | invisible() 5 | } 6 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_bad/dags_self_reference_inputs.R: -------------------------------------------------------------------------------- 1 | #' @maestroInputs with_inputs 2 | with_inputs <- function() { 3 | 4 | } 5 | 6 | 7 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_days_bad.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency hourly 2 | #' @maestroDays Mon Wed 3 | some_days <- function() { 4 | invisible() 5 | } 6 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_days_bad2.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency hourly 2 | #' @maestroDays Mon 4 3 | some_days <- function() { 4 | invisible() 5 | } 6 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_days_bad3.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency hourly 2 | #' @maestroDays 50 60 3 | some_days <- function() { 4 | invisible() 5 | } 6 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_days_good.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency hourly 2 | #' @maestroDays 1 14 3 | some_days <- function() { 4 | invisible() 5 | } 6 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_hours_bad2.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency hourly 2 | #' @maestroHours 25 3 | one_hour <- function() { 4 | invisible() 5 | } 6 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_hours_good.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency hourly 2 | #' @maestroHours 1 12 3 | one_hour <- function() { 4 | invisible() 5 | } 6 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_days_good2.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency hourly 2 | #' @maestroDays Mon Wed 3 | bad_days <- function() { 4 | invisible() 5 | } 6 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_months_bad.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency hourly 2 | #' @maestroMonths 1 30 3 | some_months <- function() { 4 | invisible() 5 | } 6 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_months_good.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency hourly 2 | #' @maestroMonths 1 6 3 | some_months <- function() { 4 | invisible() 5 | } 6 | -------------------------------------------------------------------------------- /R/zzz.R: -------------------------------------------------------------------------------- 1 | # Package environment for storing state 2 | # Currently used in error reporting scenarios like `last_build_errors` 3 | maestro_pkgenv <- new.env(parent = emptyenv()) 4 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_months_bad2.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency hourly 2 | #' @maestroMonths Jan Feb 3 | some_months <- function() { 4 | invisible() 5 | } 6 | -------------------------------------------------------------------------------- /README_cache/gfm/unnamed-chunk-2_00d877b98a39f6f6e4c275108bea86c2.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/gfm/unnamed-chunk-2_00d877b98a39f6f6e4c275108bea86c2.RData -------------------------------------------------------------------------------- /README_cache/gfm/unnamed-chunk-2_00d877b98a39f6f6e4c275108bea86c2.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/gfm/unnamed-chunk-2_00d877b98a39f6f6e4c275108bea86c2.rdb -------------------------------------------------------------------------------- /README_cache/gfm/unnamed-chunk-2_00d877b98a39f6f6e4c275108bea86c2.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/gfm/unnamed-chunk-2_00d877b98a39f6f6e4c275108bea86c2.rdx -------------------------------------------------------------------------------- /README_cache/gfm/unnamed-chunk-3_b1cda95f9f613c971ef3a8d63e3f554a.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/gfm/unnamed-chunk-3_b1cda95f9f613c971ef3a8d63e3f554a.RData -------------------------------------------------------------------------------- /README_cache/gfm/unnamed-chunk-3_b1cda95f9f613c971ef3a8d63e3f554a.cast: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/gfm/unnamed-chunk-3_b1cda95f9f613c971ef3a8d63e3f554a.cast -------------------------------------------------------------------------------- /README_cache/gfm/unnamed-chunk-3_b1cda95f9f613c971ef3a8d63e3f554a.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/gfm/unnamed-chunk-3_b1cda95f9f613c971ef3a8d63e3f554a.rdx -------------------------------------------------------------------------------- /README_cache/html/unnamed-chunk-2_91f857f630bd66849eb4330628798903.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/html/unnamed-chunk-2_91f857f630bd66849eb4330628798903.rdb -------------------------------------------------------------------------------- /README_cache/html/unnamed-chunk-2_91f857f630bd66849eb4330628798903.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/html/unnamed-chunk-2_91f857f630bd66849eb4330628798903.rdx -------------------------------------------------------------------------------- /README_cache/html/unnamed-chunk-3_73fea0572b92743b58ff926a3b43baab.cast: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/html/unnamed-chunk-3_73fea0572b92743b58ff926a3b43baab.cast -------------------------------------------------------------------------------- /README_cache/html/unnamed-chunk-3_73fea0572b92743b58ff926a3b43baab.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/html/unnamed-chunk-3_73fea0572b92743b58ff926a3b43baab.rdx -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_bad/tagged_but_no_func.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency 1 day 2 | library(dplyr) 3 | 4 | data <- iris |> 5 | dplyr::filter() |> 6 | dplyr::summarise() 7 | -------------------------------------------------------------------------------- /README_cache/html/unnamed-chunk-2_91f857f630bd66849eb4330628798903.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/html/unnamed-chunk-2_91f857f630bd66849eb4330628798903.RData -------------------------------------------------------------------------------- /README_cache/html/unnamed-chunk-3_73fea0572b92743b58ff926a3b43baab.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whipson/maestro/HEAD/README_cache/html/unnamed-chunk-3_73fea0572b92743b58ff926a3b43baab.RData -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_dup_names/dup_names.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency 1 day 2 | hello <- function() { 3 | 4 | } 5 | 6 | #' @maestroFrequency 1 day 7 | hello <- function() { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /pipelines/pipe2.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency 1 day 2 | #' @maestroStartTime 2024-03-01 09:00:00 3 | #' @maestroTz UTC 4 | #' 5 | #' @export 6 | multi_rng <- function() { 7 | replicate(10000, sample(1:100)) 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | docs 6 | maestro.log 7 | pipelines 8 | orchestrator.R 9 | inst/doc 10 | .DS_Store 11 | .Rprofile 12 | .dockerignore 13 | .Rbuildignore 14 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/MaestroSchedule.md: -------------------------------------------------------------------------------- 1 | # MaestroSchedule works 2 | 3 | Code 4 | schedule$get_network() 5 | Output 6 | # A tibble: 0 x 2 7 | # i 2 variables: from , to 8 | 9 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_bad/dags.R: -------------------------------------------------------------------------------- 1 | #' @maestroInputs fun1 fun2 fun3 2 | with_inputs <- function() { 3 | 4 | } 5 | 6 | #' @maestroOutputs fun4 fun5 7 | with_outputs <- function() { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_daily_bad.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' @maestroFrequency 14 4 | #' @maestroStartTime 5 | #' 6 | #' @export 7 | get_mtcars <- function() { 8 | mtcars 9 | } 10 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_good/pipe_with_custom_fun.R: -------------------------------------------------------------------------------- 1 | custom_fun <- function(val1, val2) { 2 | paste(val1, val2) 3 | } 4 | 5 | #' @maestroFrequency 30 minute 6 | pipe <- function() { 7 | custom_fun(1, 2) 8 | } 9 | -------------------------------------------------------------------------------- /data-raw/maestro_tags.R: -------------------------------------------------------------------------------- 1 | maestro_tags <- c( 2 | "maestroFrequency", 3 | "maestroStartTime", 4 | "maestroTz", 5 | "maestroSkip", 6 | "maestroLogLevel" 7 | ) 8 | 9 | usethis::use_data(maestro_tags, overwrite = TRUE) 10 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_all_good/chatty.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency 1 day 2 | #' @maestroStartTime 2024-03-01 09:00:00 3 | #' @maestroTz UTC 4 | #' 5 | #' @export 6 | chatty <- function() { 7 | for(i in 1:5) message(i) 8 | } 9 | -------------------------------------------------------------------------------- /inst/rstudio/templates/project/create_orchestrator.dcf: -------------------------------------------------------------------------------- 1 | Binding: create_maestro 2 | Title: Maestro Project 3 | Icon: maestro.png 4 | 5 | Parameter: type 6 | Widget: SelectInput 7 | Label: Orchestrator Type 8 | Fields: R, Quarto, RMarkdown 9 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_good/dags.R: -------------------------------------------------------------------------------- 1 | #' maestroInputs fun1 2 | my_downstream_pipe <- function(.input) { 3 | 4 | } 5 | 6 | #' maestroOutputs my_dep_pipe 7 | my_upstream_pipe <- function() { 8 | return(10) 9 | } 10 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_start_time_date.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' @maestroFrequency 4 | #' @maestroStartTime 2024-03-03 5 | #' 6 | #' @export 7 | get_mtcars <- function() { 8 | mtcars 9 | } 10 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_start_time_int.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' @maestroFrequency 4 | #' @maestroStartTime 1710254168 5 | #' 6 | #' @export 7 | get_mtcars <- function() { 8 | mtcars 9 | } 10 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_bad/test_pipeline_daily_bad.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' @maestroFrequency 17 4 | #' @maestroStartTime 5 | #' 6 | #' @export 7 | get_mtcars3 <- function() { 8 | mtcars 9 | } 10 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_daily_default.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' @maestroFrequency 4 | #' @maestroStartTime 5 | #' @maestroTz 6 | #' 7 | #' @export 8 | get_mtcars <- function() { 9 | mtcars 10 | } 11 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_tz_bad.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' @maestroFrequency 4 | #' @maestroStartTime 5 | #' @maestroTz ADT 6 | #' 7 | #' @export 8 | get_mtcars <- function() { 9 | mtcars 10 | } 11 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_some_good/test_pipeline_daily_bad.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' @maestroFrequency 17 4 | #' @maestroStartTime 5 | #' 6 | #' @export 7 | get_mtcars2 <- function() { 8 | mtcars 9 | } 10 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_bad/test_pipeline_tz_bad.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' @maestroFrequency 4 | #' @maestroStartTime 5 | #' @maestroTz ADT 6 | #' 7 | #' @export 8 | get_mtcars2 <- function() { 9 | mtcars 10 | } 11 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_artifacts/artifacts.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency 1 year 2 | artifact <- function() { 3 | 4 | return("I'm an artifact") 5 | } 6 | 7 | #' @maestroFrequency 10 days 8 | no_artifact <- function() { 9 | invisible() 10 | } 11 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_two_warnings/pipe_warning.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency 1 day 2 | pipe3 <- function() { 3 | warning("Oops") 4 | iris 5 | } 6 | 7 | #' @maestroFrequency 8 | pipe4 <- function() { 9 | warning("Another oops") 10 | rnorm(10) 11 | } 12 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_all_good/pipe2.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' @maestroFrequency 1 hour 4 | #' @maestroStartTime 2024-03-01 09:35:00 5 | #' @maestroTz UTC 6 | #' 7 | #' 8 | #' @export 9 | get_mtcars2 <- function() { 10 | mtcars 11 | } 12 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_daily_single_bad.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' @maestroFrequency minutely 4 | #' @maestroStartTime 2024-03-01 09:00:00 5 | #' @maestroTz UTC 6 | #' 7 | #' 8 | #' @export 9 | get_mtcars <- function() { 10 | mtcars 11 | } 12 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_bad/test_pipeline_no_func.R: -------------------------------------------------------------------------------- 1 | # This is just a boring script - no functions 2 | data.frame( 3 | id = 1:100, 4 | val = rnorm(100) 5 | ) 6 | 7 | # Here's a custom function, but with no tags 8 | my_fun <- function() { 9 | print("Nothing") 10 | } 11 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | comment: false 2 | 3 | coverage: 4 | status: 5 | project: 6 | default: 7 | target: auto 8 | threshold: 1% 9 | informational: true 10 | patch: 11 | default: 12 | target: auto 13 | threshold: 1% 14 | informational: true 15 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_skip.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroSkip 8 | #' 9 | #' @export 10 | get_mtcars <- function() { 11 | mtcars 12 | } 13 | -------------------------------------------------------------------------------- /inst/pipeline_template: -------------------------------------------------------------------------------- 1 | #' {{pipe_name}} maestro pipeline 2 | #' 3 | #' @maestroFrequency {{frequency}} 4 | #' @maestroStartTime {{start_time}} 5 | #' @maestroTz {{tz}} 6 | #' @maestroLogLevel {{log_level}}{{priority}}{{inputs}}{{outputs}}{{skip}} 7 | 8 | {{pipe_name}} <- function() { 9 | 10 | # Pipeline code 11 | } 12 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_skip_bad.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroSkip some content 8 | #' 9 | #' @export 10 | get_mtcars <- function() { 11 | mtcars 12 | } 13 | -------------------------------------------------------------------------------- /inst/sample_pipeline_dir/pipe2.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' 11 | #' @export 12 | get_mtcars2 <- function() { 13 | mtcars 14 | } 15 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_loglevel_bad.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroLogLevel Ghost 8 | #' 9 | #' @export 10 | get_mtcars <- function() { 11 | Sys.sleep(0.02) 12 | mtcars 13 | } 14 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_loglevel_good.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroLogLevel INFO 8 | #' 9 | #' @export 10 | get_mtcars <- function() { 11 | Sys.sleep(0.02) 12 | mtcars 13 | } 14 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_dags_run_bad/dag_with_error.R: -------------------------------------------------------------------------------- 1 | # Simple 1 -> 1 ----------------------------------------------------------- 2 | #' @maestroInputs get_num 3 | multiply <- function(.input) { 4 | .input * 4 5 | } 6 | 7 | #' @maestroFrequency daily 8 | #' @maestroOutputs multiply 9 | get_num <- function() { 10 | stop("oops") 11 | } 12 | -------------------------------------------------------------------------------- /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^renv$ 2 | ^renv\.lock$ 3 | ^.*\.Rproj$ 4 | ^\.Rproj\.user$ 5 | ^README\.qmd$ 6 | ^data-raw$ 7 | ^LICENSE\.md$ 8 | ^_pkgdown\.yml$ 9 | ^docs$ 10 | ^pkgdown$ 11 | ^\.github$ 12 | ^README_cache 13 | ^maestro.log 14 | ^orchestrator.R 15 | ^pipelines$ 16 | ^cran-comments\.md$ 17 | ^codecov\.yml$ 18 | ^CRAN-SUBMISSION$ 19 | ^\.dockerignore$ 20 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_logs_warn/pipe_warning.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency 1 day 2 | #' @maestroLogLevel WARN 3 | pipe3 <- function() { 4 | message("Hide me") 5 | 6 | warning("Oops") 7 | iris 8 | } 9 | 10 | #' @maestroLogLevel WARN 11 | pipe4 <- function() { 12 | message("Hide me too") 13 | 14 | warning("Another oops") 15 | rnorm(10) 16 | } 17 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_good/pipe2.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' 11 | #' @export 12 | get_mtcars2 <- function() { 13 | mtcars 14 | } 15 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_two_warnings/pipe2.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' 11 | #' @export 12 | get_mtcars2 <- function() { 13 | mtcars 14 | } 15 | -------------------------------------------------------------------------------- /pipelines/pipe1.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' @export 11 | get_mtcars <- function() { 12 | Sys.sleep(0.02) 13 | warning("Uh oh") 14 | mtcars 15 | } 16 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_daily_good.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' 11 | #' @export 12 | get_mtcars <- function() { 13 | mtcars 14 | } 15 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_bad/tagged_but_no_func_multi.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency 2 | data <- iris |> 3 | dplyr::filter() |> 4 | dplyr::summarise() 5 | 6 | #' @maestroFrequency 7 | data |> 8 | head(n = 6) 9 | 10 | #' @maestroFrequency 11 | this_one_is_good <- function() { 12 | # But the fact that there are bad ones above makes it bad 13 | iris 14 | } 15 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_pipeline_daily_single_good.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency daily 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' 11 | #' @export 12 | get_mtcars <- function() { 13 | mtcars 14 | } 15 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_some_good/test_pipeline_daily_good.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' 11 | #' @export 12 | get_mtcars <- function() { 13 | mtcars 14 | } 15 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_args_good/pipe1.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' 11 | #' @export 12 | get_mtcars2 <- function(vals) { 13 | if (is.null(vals)) stop("Null vals") 14 | mean(vals) 15 | } 16 | -------------------------------------------------------------------------------- /man/is_valid_dag.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{is_valid_dag} 4 | \alias{is_valid_dag} 5 | \title{Checks whether a DAG is valid (no loops)} 6 | \usage{ 7 | is_valid_dag(edges) 8 | } 9 | \arguments{ 10 | \item{edges}{a data.frame of edges (from, to)} 11 | } 12 | \value{ 13 | boolean 14 | } 15 | \description{ 16 | Checks whether a DAG is valid (no loops) 17 | } 18 | \keyword{internal} 19 | -------------------------------------------------------------------------------- /man/parse_rounding_unit.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{parse_rounding_unit} 4 | \alias{parse_rounding_unit} 5 | \title{Parse a time string} 6 | \usage{ 7 | parse_rounding_unit(time_string) 8 | } 9 | \arguments{ 10 | \item{time_string}{string like 1 day, daily, 1 week, 12 hours, etc.} 11 | } 12 | \value{ 13 | nunit list 14 | } 15 | \description{ 16 | Parse a time string 17 | } 18 | \keyword{internal} 19 | -------------------------------------------------------------------------------- /tests/testthat.R: -------------------------------------------------------------------------------- 1 | # This file is part of the standard setup for testthat. 2 | # It is recommended that you do not modify it. 3 | # 4 | # Where should you do additional test configuration? 5 | # Learn more about the roles of various files in: 6 | # * https://r-pkgs.org/testing-design.html#sec-tests-files-overview 7 | # * https://testthat.r-lib.org/articles/special-files.html 8 | 9 | library(testthat) 10 | library(dplyr) 11 | library(maestro) 12 | library(future) 13 | 14 | test_check("maestro") 15 | -------------------------------------------------------------------------------- /inst/sample_pipeline_dir/pipe1.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' @export 11 | get_mtcars <- function() { 12 | Sys.sleep(0.02) 13 | mtcars 14 | } 15 | 16 | #' Multiply 17 | #' 18 | #' @maestroFrequency 1 month 19 | #' 20 | #' @export 21 | wait <- function() { 22 | Sys.sleep(0.01) 23 | } 24 | -------------------------------------------------------------------------------- /maestro.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | ProjectId: 58cd0b4c-72f2-4b6c-8e04-a17f7a841c7f 3 | 4 | RestoreWorkspace: Default 5 | SaveWorkspace: Default 6 | AlwaysSaveHistory: Default 7 | 8 | EnableCodeIndexing: Yes 9 | UseSpacesForTab: Yes 10 | NumSpacesForTab: 2 11 | Encoding: UTF-8 12 | 13 | RnwWeave: Sweave 14 | LaTeX: pdfLaTeX 15 | 16 | AutoAppendNewline: Yes 17 | StripTrailingWhitespace: Yes 18 | 19 | BuildType: Package 20 | PackageUseDevtools: Yes 21 | PackageInstallArgs: --no-multiarch --with-keep.source 22 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_some_errors/pipe2.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' @export 11 | lm_mtcars <- function() { 12 | lm("data") 13 | } 14 | 15 | #' Multiply 16 | #' 17 | #' @maestroFrequency 3 month 18 | #' 19 | #' @export 20 | wait2 <- function() { 21 | Sys.sleep(0.01) 22 | } 23 | -------------------------------------------------------------------------------- /renv/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "bioconductor.version": null, 3 | "external.libraries": [], 4 | "ignored.packages": [], 5 | "package.dependency.fields": [ 6 | "Imports", 7 | "Depends", 8 | "LinkingTo" 9 | ], 10 | "ppm.enabled": null, 11 | "ppm.ignored.urls": [], 12 | "r.version": null, 13 | "snapshot.type": "explicit", 14 | "use.cache": true, 15 | "vcs.ignore.cellar": true, 16 | "vcs.ignore.library": true, 17 | "vcs.ignore.local": true, 18 | "vcs.manage.ignores": true 19 | } 20 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_multi_fun_pipeline.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' @export 11 | get_mtcars <- function() { 12 | mtcars 13 | } 14 | 15 | #' Multiply 16 | #' 17 | #' @maestroFrequency 3 months 18 | #' 19 | #' @export 20 | multiply <- function(val, by) { 21 | val * by 22 | } 23 | -------------------------------------------------------------------------------- /man/convert_to_seconds.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{convert_to_seconds} 4 | \alias{convert_to_seconds} 5 | \title{Convert a duration string to number of seconds} 6 | \usage{ 7 | convert_to_seconds(time_string) 8 | } 9 | \arguments{ 10 | \item{time_string}{string like 1 day, 1 week, 12 hours, etc.} 11 | } 12 | \value{ 13 | number of seconds 14 | } 15 | \description{ 16 | Convert a duration string to number of seconds 17 | } 18 | \keyword{internal} 19 | -------------------------------------------------------------------------------- /pkgdown/_pkgdown.yml: -------------------------------------------------------------------------------- 1 | url: https://whipson.github.io/maestro/ 2 | template: 3 | bootstrap: 5 4 | bslib: 5 | primary: "#9012C7" 6 | 7 | news: 8 | releases: 9 | - text: "0.4.0" 10 | href: https://whipson.github.io/data-in-flight/posts/maestro-dags/main.html 11 | - text: "0.6.0" 12 | href: https://whipson.github.io/data-in-flight/posts/maestro-0-6-0/maestro-0-6-0-release.html 13 | - text: "0.7.0" 14 | href: https://whipson.github.io/data-in-flight/posts/maestro-0-7-0/maestro-0-7-0-release.html 15 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_some_errors/pipe1.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' @export 11 | get_mtcars <- function() { 12 | Sys.sleep(0.02) 13 | mtcars 14 | } 15 | 16 | #' Multiply 17 | #' 18 | #' @maestroFrequency 3 month 19 | #' 20 | #' @export 21 | wait <- function() { 22 | Sys.sleep(0.01) 23 | } 24 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_two_warnings/pipe1.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' @export 11 | get_mtcars <- function() { 12 | Sys.sleep(0.02) 13 | mtcars 14 | } 15 | 16 | #' Multiply 17 | #' 18 | #' @maestroFrequency 3 month 19 | #' 20 | #' @export 21 | wait <- function() { 22 | Sys.sleep(0.01) 23 | } 24 | -------------------------------------------------------------------------------- /tests/testthat/test-get_status.R: -------------------------------------------------------------------------------- 1 | test_that("get_status returns a data.frame", { 2 | schedule <- build_schedule(test_path("test_pipelines_run_all_good"), quiet = TRUE) 3 | schedule <- run_schedule(schedule, orch_frequency = "hourly") 4 | 5 | expect_s3_class( 6 | get_status(schedule), 7 | "data.frame" 8 | ) 9 | }) |> 10 | suppressMessages() 11 | 12 | test_that("errors if schedule is not a MaestroSchedule", { 13 | expect_error({ 14 | get_status(iris) 15 | }, regexp = "Schedule must be an object") 16 | }) 17 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/MaestroPipelineList.md: -------------------------------------------------------------------------------- 1 | # Simple pipeline list, no errors 2 | 3 | Code 4 | pipeline_list 5 | Message 6 | 7 | -- Maestro Pipelines List with 1 pipeline 8 | 9 | --- 10 | 11 | Code 12 | pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")] 13 | Output 14 | # A tibble: 1 x 5 15 | invoked success errors warnings messages 16 | 17 | 1 TRUE TRUE 0 0 0 18 | 19 | -------------------------------------------------------------------------------- /man/last_run_errors.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/latest_getters.R 3 | \name{last_run_errors} 4 | \alias{last_run_errors} 5 | \title{Retrieve latest maestro pipeline errors} 6 | \usage{ 7 | last_run_errors() 8 | } 9 | \value{ 10 | error messages 11 | } 12 | \description{ 13 | Gets the latest pipeline errors following use of \code{run_schedule()}. If 14 | the all runs succeeded or \code{run_schedule()} has not been run it will be \code{NULL}. 15 | } 16 | \examples{ 17 | 18 | last_run_errors() 19 | } 20 | -------------------------------------------------------------------------------- /pipelines/my_etl.R: -------------------------------------------------------------------------------- 1 | #' Example ETL pipeline 2 | #' @maestroFrequency 1 day 3 | #' @maestroStartTime 2024-03-25 12:30:00 4 | my_etl <- function() { 5 | 6 | # Pretend we're getting data from a source 7 | message("Get data") 8 | extracted <- mtcars 9 | 10 | # Transform 11 | message("Transforming") 12 | transformed <- extracted |> 13 | dplyr::mutate(hp_deviation = hp - mean(hp)) 14 | 15 | # Load - write to a location 16 | message("Writing") 17 | # write.csv(transformed, file = paste0("transformed_mtcars_", Sys.Date(), ".csv")) 18 | } 19 | -------------------------------------------------------------------------------- /man/last_build_errors.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/latest_getters.R 3 | \name{last_build_errors} 4 | \alias{last_build_errors} 5 | \title{Retrieve latest maestro build errors} 6 | \usage{ 7 | last_build_errors() 8 | } 9 | \value{ 10 | error messages 11 | } 12 | \description{ 13 | Gets the latest schedule build errors following use of \code{build_schedule()}. If 14 | the build succeeded or \code{build_schedule()} has not been run it will be \code{NULL}. 15 | } 16 | \examples{ 17 | 18 | last_build_errors() 19 | } 20 | -------------------------------------------------------------------------------- /man/maestro_parse_cli.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils_cli.R 3 | \name{maestro_parse_cli} 4 | \alias{maestro_parse_cli} 5 | \title{cli output for generate schedule table} 6 | \usage{ 7 | maestro_parse_cli(parse_succeeds, parse_errors) 8 | } 9 | \arguments{ 10 | \item{parse_succeeds}{list of parse results (i.e., succeeded)} 11 | 12 | \item{parse_errors}{list of parse errors} 13 | } 14 | \value{ 15 | cli output 16 | } 17 | \description{ 18 | cli output for generate schedule table 19 | } 20 | \keyword{internal} 21 | -------------------------------------------------------------------------------- /man/build_schedule_entry.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/build_schedule_entry.R 3 | \name{build_schedule_entry} 4 | \alias{build_schedule_entry} 5 | \title{Parse and validate tags then create and populate MaestroPipelineList} 6 | \usage{ 7 | build_schedule_entry(script_path) 8 | } 9 | \arguments{ 10 | \item{script_path}{path to script} 11 | } 12 | \value{ 13 | MaestroPipelineList R6 class 14 | } 15 | \description{ 16 | Parse and validate tags then create and populate MaestroPipelineList 17 | } 18 | \keyword{internal} 19 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/invoke.md: -------------------------------------------------------------------------------- 1 | # invoke gives informative error message on failure 2 | 3 | Code 4 | schedule$get_status()[, c("invoked", "success")] 5 | Output 6 | # A tibble: 1 x 2 7 | invoked success 8 | 9 | 1 TRUE FALSE 10 | 11 | # invoke triggers DAG pipelines 12 | 13 | Code 14 | schedule$get_status()[, c("invoked", "success")] 15 | Output 16 | # A tibble: 2 x 2 17 | invoked success 18 | 19 | 1 TRUE TRUE 20 | 2 TRUE TRUE 21 | 22 | -------------------------------------------------------------------------------- /tests/testthat/test-get_artifacts.R: -------------------------------------------------------------------------------- 1 | test_that("get_artifacts returns artifacts", { 2 | schedule <- build_schedule(test_path("test_pipelines_run_artifacts")) 3 | 4 | output <- run_schedule( 5 | schedule, 6 | run_all = TRUE 7 | ) 8 | 9 | artifacts <- get_artifacts(output) 10 | 11 | expect_type(artifacts, "list") 12 | expect_gt(length(artifacts), 0) 13 | }) |> 14 | suppressMessages() 15 | 16 | test_that("errors if schedule is not a MaestroSchedule", { 17 | expect_error({ 18 | get_artifacts(iris) 19 | }, regexp = "Schedule must be an object") 20 | }) 21 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/get_flags.md: -------------------------------------------------------------------------------- 1 | # get_flags works as expected 2 | 3 | Code 4 | flags_df 5 | Output 6 | # A tibble: 5 x 2 7 | pipe_name flag 8 | 9 | 1 p1 experimental 10 | 2 p1 dev 11 | 3 p2 critical 12 | 4 p2 aviation 13 | 5 p3 critical 14 | 15 | # returns empty data.frame if there are no flags 16 | 17 | Code 18 | flags_df 19 | Output 20 | # A tibble: 0 x 2 21 | # i 2 variables: pipe_name , flag 22 | 23 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines/test_multi_fun_pipeline_one_bad.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' @export 11 | get_mtcars <- function() { 12 | mtcars 13 | } 14 | 15 | #' Multiply 16 | #' 17 | #' @maestroFrequency 3 months 18 | #' 19 | #' @export 20 | multiply <- function(val, by) { 21 | val * by 22 | } 23 | 24 | #' Bad 25 | #' 26 | #' @maestroTz BLA 27 | something_else <- function() { 28 | 29 | } 30 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_bad/specifiers.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency weekly 2 | #' @maestroDays 1 8 24 28 3 | specific_days <- function() { 4 | 5 | } 6 | 7 | #' @maestroFrequency daily 8 | #' @maestroHours 0 4 8 12 16 20 9 | specific_hours <- function() { 10 | 11 | } 12 | 13 | #' @maestroFrequency yearly 14 | #' @maestroMonths 1 5 10 15 | specific_months1 <- function() { 16 | 17 | } 18 | 19 | #' @maestroFrequency quarterly 20 | #' @maestroMonths 1 5 10 21 | specific_months2 <- function() { 22 | 23 | } 24 | 25 | #' @maestroFrequency daily 26 | #' @maestroHours 1 12 27 | #' @maestroDays 1 28 | #' @maestroMonths 1 5 10 29 | specific_multi <- function() { 30 | 31 | } 32 | -------------------------------------------------------------------------------- /man/last_run_messages.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/latest_getters.R 3 | \name{last_run_messages} 4 | \alias{last_run_messages} 5 | \title{Retrieve latest maestro pipeline messages} 6 | \usage{ 7 | last_run_messages() 8 | } 9 | \value{ 10 | messages 11 | } 12 | \description{ 13 | Gets the latest pipeline messages following use of \code{run_schedule()}. If 14 | there are no messages or \code{run_schedule()} has not been run it will be \code{NULL}. 15 | } 16 | \details{ 17 | Note that setting \code{maestroLogLevel} to something greater than \code{INFO} will 18 | ignore messages. 19 | } 20 | \examples{ 21 | 22 | last_run_messages() 23 | } 24 | -------------------------------------------------------------------------------- /man/last_run_warnings.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/latest_getters.R 3 | \name{last_run_warnings} 4 | \alias{last_run_warnings} 5 | \title{Retrieve latest maestro pipeline warnings} 6 | \usage{ 7 | last_run_warnings() 8 | } 9 | \value{ 10 | warning messages 11 | } 12 | \description{ 13 | Gets the latest pipeline warnings following use of \code{run_schedule()}. If 14 | there are no warnings or \code{run_schedule()} has not been run it will be \code{NULL}. 15 | } 16 | \details{ 17 | Note that setting \code{maestroLogLevel} to something greater than \code{WARN} will 18 | ignore warnings. 19 | } 20 | \examples{ 21 | 22 | last_run_warnings() 23 | } 24 | -------------------------------------------------------------------------------- /tests/testthat/test-show_network.R: -------------------------------------------------------------------------------- 1 | test_that("show_network works on all independent pipes", { 2 | schedule <- build_schedule(test_path("test_pipelines_run_all_good"), quiet = TRUE) 3 | vis <- show_network(schedule) 4 | expect_s3_class(vis, "grViz") 5 | }) |> 6 | suppressMessages() 7 | 8 | test_that("show_network works on DAG pipelines", { 9 | schedule <- build_schedule(test_path("test_pipelines_dags_good"), quiet = TRUE) 10 | vis <- show_network(schedule) 11 | expect_s3_class(vis, "grViz") 12 | }) |> 13 | suppressMessages() 14 | 15 | test_that("errors if schedule is not a MaestroSchedule", { 16 | expect_error({ 17 | show_network(iris) 18 | }, regexp = "Schedule must be an object") 19 | }) 20 | -------------------------------------------------------------------------------- /R/maestro.R: -------------------------------------------------------------------------------- 1 | #' \code{maestro} package 2 | #' 3 | #' Lightweight pipeline orchestration in R 4 | #' 5 | #' Documentation: \href{https://github.com/whipson/maestro}{GitHub} 6 | #' @keywords internal 7 | #' @name maestro 8 | "_PACKAGE" 9 | 10 | ## quiets concerns of R CMD check re: the .'s that appear in pipelines 11 | if(getRversion() >= "2.15.1") utils::globalVariables( 12 | c("frequency", "start_time", "tz", "skip", "log_level", "is_scheduled_now", 13 | "next_run", "frequency_n", "frequency_unit", "errors", "pipe_name", 14 | "script_path", "invoked", "success", "pipeline_started", "pipeline_ended", 15 | "messages", "run_time", "run_date", "run_time_15min", "n_runs", 16 | "hours", "days_of_week", "days_of_month", "days", "months", "slot") 17 | ) 18 | -------------------------------------------------------------------------------- /tests/testthat/test-create_orchestrator.R: -------------------------------------------------------------------------------- 1 | test_that("create_orchestrator creates a new orchestrator", { 2 | withr::with_tempdir({ 3 | create_orchestrator("orchestrator" ,open = FALSE) 4 | expect_true(file.exists("orchestrator.R")) 5 | }) |> 6 | expect_message() 7 | }) 8 | 9 | test_that("create_orchestrator aborts if file already exists", { 10 | 11 | withr::with_tempdir({ 12 | expect_error({ 13 | create_orchestrator(".", open = FALSE) 14 | create_orchestrator(".", open = FALSE) 15 | }, regexp = "already exists.") 16 | 17 | # Works if overwrite = TRUE 18 | expect_message( 19 | create_orchestrator(".", open = FALSE, overwrite = TRUE), 20 | regexp = "Overwriting existing" 21 | ) 22 | }) 23 | }) |> 24 | suppressMessages() 25 | -------------------------------------------------------------------------------- /tests/testthat/test-get_schedule.R: -------------------------------------------------------------------------------- 1 | test_that("get_schedule returns a data.frame", { 2 | schedule <- build_schedule(test_path("test_pipelines_run_all_good"), quiet = TRUE) 3 | expect_s3_class( 4 | get_schedule(schedule), 5 | "data.frame" 6 | ) 7 | 8 | expect_snapshot(schedule) 9 | }) |> 10 | suppressMessages() 11 | 12 | test_that("errors if schedule is not a MaestroSchedule", { 13 | expect_error({ 14 | get_schedule(iris) 15 | }, regexp = "Schedule must be an object") 16 | }) 17 | 18 | test_that("get_schedule works with DAG schedules", { 19 | schedule <- build_schedule(test_path("test_pipelines_dags_good")) 20 | expect_snapshot(get_schedule(schedule)[, c("script_path", "pipe_name", "frequency", "tz", "skip", "log_level")]) 21 | }) |> 22 | suppressMessages() 23 | -------------------------------------------------------------------------------- /man/get_flags.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get_flags.R 3 | \name{get_flags} 4 | \alias{get_flags} 5 | \title{Get the flags of pipelines in a MaestroSchedule object} 6 | \usage{ 7 | get_flags(schedule) 8 | } 9 | \arguments{ 10 | \item{schedule}{object of type MaestroSchedule created using \code{build_schedule()}} 11 | } 12 | \value{ 13 | data.frame 14 | } 15 | \description{ 16 | Creates a long data.frame where each row is a flag for each pipeline. 17 | } 18 | \examples{ 19 | if (interactive()) { 20 | pipeline_dir <- tempdir() 21 | create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 22 | schedule <- build_schedule(pipeline_dir = pipeline_dir) 23 | 24 | get_flags(schedule) 25 | 26 | # Alternatively, use the underlying R6 method 27 | schedule$get_flags() 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/get_schedule.md: -------------------------------------------------------------------------------- 1 | # get_schedule returns a data.frame 2 | 3 | Code 4 | schedule 5 | Message 6 | 7 | -- Maestro Schedule with 7 pipelines: 8 | * Not Run 9 | 10 | # get_schedule works with DAG schedules 11 | 12 | Code 13 | get_schedule(schedule)[, c("script_path", "pipe_name", "frequency", "tz", 14 | "skip", "log_level")] 15 | Output 16 | # A tibble: 3 x 6 17 | script_path pipe_name frequency tz skip log_level 18 | 19 | 1 test_pipelines_dags_good/dags.R primary daily UTC FALSE INFO 20 | 2 test_pipelines_dags_good/dags.R trunk daily UTC FALSE INFO 21 | 3 test_pipelines_dags_good/dags.R start daily UTC FALSE INFO 22 | 23 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_all_good/pipe1.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz America/Halifax 9 | #' 10 | #' @export 11 | get_mtcars <- function() { 12 | Sys.sleep(0.02) 13 | mtcars 14 | } 15 | 16 | #' Multiply 17 | #' 18 | #' @maestroFrequency 3 month 19 | #' 20 | #' @export 21 | wait <- function() { 22 | Sys.sleep(0.01) 23 | } 24 | 25 | #' @maestroFrequency 1 week 26 | #' 27 | #' @export 28 | weekly <- function() { 29 | 1 + 1 30 | } 31 | 32 | #' @maestroStartTime 2030-12-12 10:10:10 33 | #' @maestroFrequency 1 day 34 | way_in_the_future <- function() { 35 | invisible() 36 | } 37 | 38 | #' @maestroFrequency weekly 39 | #' 40 | #' @export 41 | weekly2 <- function() { 42 | 1 + 1 43 | } 44 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_good/specifiers.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency daily 2 | #' @maestroDays 1 8 24 28 3 | specific_days <- function() { 4 | 5 | } 6 | 7 | #' @maestroFrequency daily 8 | #' @maestroDays Tue Fri 9 | specific_days2 <- function() { 10 | 11 | } 12 | 13 | #' @maestroFrequency hourly 14 | #' @maestroHours 0 4 8 12 16 20 15 | specific_hours <- function() { 16 | 17 | } 18 | 19 | #' @maestroFrequency biweekly 20 | #' @maestroMonths 1 5 10 21 | specific_months1 <- function() { 22 | 23 | } 24 | 25 | #' @maestroFrequency weekly 26 | #' @maestroMonths 1 5 10 27 | specific_months2 <- function() { 28 | 29 | } 30 | 31 | #' @maestroFrequency monthly 32 | #' @maestroMonths 1 5 10 33 | specific_months3 <- function() { 34 | 35 | } 36 | 37 | #' @maestroFrequency hourly 38 | #' @maestroHours 1 12 39 | #' @maestroDays 1 40 | #' @maestroMonths 1 5 10 41 | specific_multi <- function() { 42 | 43 | } 44 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_specifiers/specifiers.R: -------------------------------------------------------------------------------- 1 | #' @maestroFrequency daily 2 | #' @maestroDays 1 8 24 28 3 | specific_days <- function() { 4 | 5 | } 6 | 7 | #' @maestroFrequency hourly 8 | #' @maestroHours 0 4 8 12 16 20 9 | specific_hours <- function() { 10 | 11 | } 12 | 13 | #' @maestroFrequency biweekly 14 | #' @maestroMonths 1 5 10 15 | specific_months1 <- function() { 16 | 17 | } 18 | 19 | #' @maestroFrequency hourly 20 | #' @maestroDays Mon Wed Fri 21 | specific_days2 <- function() { 22 | 23 | } 24 | 25 | #' @maestroFrequency hourly 26 | #' @maestroDays Sun Sat 27 | specific_days3 <- function() { 28 | 29 | } 30 | 31 | #' @maestroFrequency monthly 32 | #' @maestroMonths 1 5 10 33 | specific_months3 <- function() { 34 | 35 | } 36 | 37 | #' @maestroFrequency hourly 38 | #' @maestroHours 1 12 39 | #' @maestroDays 1 40 | #' @maestroMonths 1 5 10 41 | specific_multi <- function() { 42 | 43 | } 44 | -------------------------------------------------------------------------------- /man/create_orchestrator.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/create_orchestrator.R 3 | \name{create_orchestrator} 4 | \alias{create_orchestrator} 5 | \title{Create a new orchestrator} 6 | \usage{ 7 | create_orchestrator( 8 | path, 9 | type = c("R", "Quarto", "RMarkdown"), 10 | open = interactive(), 11 | quiet = FALSE, 12 | overwrite = FALSE 13 | ) 14 | } 15 | \arguments{ 16 | \item{path}{file path for the orchestrator script} 17 | 18 | \item{type}{file type for the orchestrator (supports R, Quarto, and RMarkdown)} 19 | 20 | \item{open}{whether or not to open the script upon creation} 21 | 22 | \item{quiet}{whether to silence messages in the console (default = \code{FALSE})} 23 | 24 | \item{overwrite}{whether to overwrite an existing orchestrator or maestro project} 25 | } 26 | \value{ 27 | invisible 28 | } 29 | \description{ 30 | Create a new orchestrator 31 | } 32 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_run_skip/with_skip.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' 10 | #' @export 11 | get_mtcars <- function() { 12 | Sys.sleep(0.02) 13 | mtcars 14 | } 15 | 16 | #' Multiply 17 | #' 18 | #' @maestroFrequency 3 month 19 | #' @maestroStartTime 1970-01-01 20 | #' @maestroSkip 21 | #' @maestroTz UTC 22 | #' 23 | #' @export 24 | wait <- function() { 25 | Sys.sleep(0.01) 26 | } 27 | 28 | #' Simple mtcars print function 29 | #' 30 | #' This is a function that runs every hour starting at 31 | #' 2024-03-01 09:00:00 32 | #' 33 | #' @maestroFrequency 1 day 34 | #' @maestroStartTime 2024-03-01 09:00:00 35 | #' @maestroTz UTC 36 | #' 37 | #' @export 38 | get_mtcars2 <- function() { 39 | Sys.sleep(0.02) 40 | mtcars 41 | } 42 | -------------------------------------------------------------------------------- /tests/testthat/test-suggest_orch_frequency.R: -------------------------------------------------------------------------------- 1 | example_schedule <- build_schedule(test_path("test_pipelines_parse_all_good"), quiet = TRUE) 2 | 3 | test_that("suggest_orch_frequency gives valid suggestions", { 4 | 5 | expect_equal( 6 | suggest_orch_frequency(example_schedule), 7 | "30 mins" 8 | ) 9 | }) 10 | 11 | test_that("suggest_orch_frequency works when check_datetime is a Date", { 12 | expect_no_error( 13 | suggest_orch_frequency(example_schedule, check_datetime = as.Date("2024-08-06")) 14 | ) 15 | }) 16 | 17 | test_that("suggest_orch_frequency gives expected errors", { 18 | expect_error({ 19 | suggest_orch_frequency(1) 20 | }, regexp = "Schedule must be an object") 21 | }) 22 | 23 | test_that("suggest_orch_frequency works with DAG schedules", { 24 | schedule <- build_schedule(test_path("test_pipelines_dags_good")) 25 | expect_true(!is.na(suggest_orch_frequency(schedule))) 26 | }) |> 27 | suppressMessages() 28 | -------------------------------------------------------------------------------- /man/maestro.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/maestro.R 3 | \docType{package} 4 | \name{maestro} 5 | \alias{maestro-package} 6 | \alias{maestro} 7 | \title{\code{maestro} package} 8 | \description{ 9 | Lightweight pipeline orchestration in R 10 | } 11 | \details{ 12 | Documentation: \href{https://github.com/whipson/maestro}{GitHub} 13 | } 14 | \seealso{ 15 | Useful links: 16 | \itemize{ 17 | \item \url{https://github.com/whipson/maestro} 18 | \item \url{https://whipson.github.io/maestro/} 19 | \item Report bugs at \url{https://github.com/whipson/maestro/issues} 20 | } 21 | 22 | } 23 | \author{ 24 | \strong{Maintainer}: Will Hipson \email{will.e.hipson@gmail.com} (\href{https://orcid.org/0000-0002-3931-2189}{ORCID}) [copyright holder] 25 | 26 | Authors: 27 | \itemize{ 28 | \item Ryan Garnett \email{ryangarnett78@gmail.com} [contributor, copyright holder] 29 | } 30 | 31 | } 32 | \keyword{internal} 33 | -------------------------------------------------------------------------------- /R/get_flags.R: -------------------------------------------------------------------------------- 1 | #' Get the flags of pipelines in a MaestroSchedule object 2 | #' 3 | #' Creates a long data.frame where each row is a flag for each pipeline. 4 | #' 5 | #' @inheritParams run_schedule 6 | #' 7 | #' @return data.frame 8 | #' @export 9 | #' @examples 10 | #' if (interactive()) { 11 | #' pipeline_dir <- tempdir() 12 | #' create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 13 | #' schedule <- build_schedule(pipeline_dir = pipeline_dir) 14 | #' 15 | #' get_flags(schedule) 16 | #' 17 | #' # Alternatively, use the underlying R6 method 18 | #' schedule$get_flags() 19 | #' } 20 | get_flags <- function(schedule) { 21 | 22 | if (!"MaestroSchedule" %in% class(schedule)) { 23 | cli::cli_abort( 24 | c("Schedule must be an object of {.cls MaestroSchedule} and not an object of class {.cls {class(schedule)}}.", 25 | "i" = "Use {.fn build_schedule} to create a valid schedule."), 26 | call = rlang::caller_env() 27 | ) 28 | } 29 | 30 | schedule$get_flags() 31 | } 32 | -------------------------------------------------------------------------------- /tests/testthat/test-create_maestro.R: -------------------------------------------------------------------------------- 1 | test_that("create_maestro creates a new maestro project", { 2 | withr::with_tempdir({ 3 | expect_message(create_maestro(path = ".", type = "R", overwrite = TRUE)) 4 | expect_true(file.exists("orchestrator.R")) 5 | expect_true(dir.exists("pipelines")) 6 | }) 7 | 8 | withr::with_tempdir({ 9 | expect_message(create_maestro(path = ".", type = "Quarto", overwrite = TRUE)) 10 | expect_true(file.exists("orchestrator.qmd")) 11 | expect_true(dir.exists("pipelines")) 12 | }) 13 | }) |> 14 | suppressMessages() 15 | 16 | test_that("create_maestro aborts if directory already exists", { 17 | withr::with_tempdir({ 18 | expect_error( 19 | create_maestro(path = "."), 20 | regexp = "Project directory already exists" 21 | ) 22 | }) 23 | 24 | # If overwrite = TRUE it will work 25 | withr::with_tempdir({ 26 | expect_message( 27 | create_maestro(path = ".", overwrite = TRUE), 28 | regexp = "Overwriting existing project" 29 | ) 30 | }) 31 | }) |> 32 | suppressMessages() 33 | -------------------------------------------------------------------------------- /man/create_maestro.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/create_maestro.R 3 | \name{create_maestro} 4 | \alias{create_maestro} 5 | \title{Creates a new maestro project} 6 | \usage{ 7 | create_maestro(path, type = "R", overwrite = FALSE, quiet = FALSE, ...) 8 | } 9 | \arguments{ 10 | \item{path}{file path for the orchestrator script} 11 | 12 | \item{type}{file type for the orchestrator (supports R, Quarto, and RMarkdown)} 13 | 14 | \item{overwrite}{whether to overwrite an existing orchestrator or maestro project} 15 | 16 | \item{quiet}{whether to silence messages in the console (default = \code{FALSE})} 17 | 18 | \item{...}{unused} 19 | } 20 | \value{ 21 | invisible 22 | } 23 | \description{ 24 | Creates a new maestro project 25 | } 26 | \examples{ 27 | 28 | # Creates a new maestro project with an R orchestrator 29 | if (interactive()) { 30 | new_proj_dir <- tempdir() 31 | create_maestro(new_proj_dir, type = "R", overwrite = TRUE) 32 | 33 | create_maestro(new_proj_dir, type = "Quarto", overwrite = TRUE) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /man/get_artifacts.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get_artifacts.R 3 | \name{get_artifacts} 4 | \alias{get_artifacts} 5 | \title{Get the artifacts (return values) of the pipelines in a MaestroSchedule object.} 6 | \usage{ 7 | get_artifacts(schedule) 8 | } 9 | \arguments{ 10 | \item{schedule}{object of type MaestroSchedule created using \code{build_schedule()}} 11 | } 12 | \value{ 13 | named list 14 | } 15 | \description{ 16 | Artifacts are return values from pipelines. They are accessible as a named list 17 | where the names correspond to the names of the pipeline. 18 | } 19 | \examples{ 20 | if (interactive()) { 21 | pipeline_dir <- tempdir() 22 | create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 23 | schedule <- build_schedule(pipeline_dir = pipeline_dir) 24 | 25 | schedule <- run_schedule( 26 | schedule, 27 | orch_frequency = "1 day", 28 | quiet = TRUE 29 | ) 30 | 31 | get_artifacts(schedule) 32 | 33 | # Alternatively, use the underlying R6 method 34 | schedule$get_artifacts() 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /man/get_status.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get_status.R 3 | \name{get_status} 4 | \alias{get_status} 5 | \title{Get the statuses of the pipelines in a MaestroSchedule object} 6 | \usage{ 7 | get_status(schedule) 8 | } 9 | \arguments{ 10 | \item{schedule}{object of type MaestroSchedule created using \code{build_schedule()}} 11 | } 12 | \value{ 13 | data.frame 14 | } 15 | \description{ 16 | A status data.frame contains the names and locations of the pipelines as 17 | well as information around whether they were invoked, the status (error, warning, etc.), 18 | and the run time. 19 | } 20 | \examples{ 21 | if (interactive()) { 22 | pipeline_dir <- tempdir() 23 | create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 24 | schedule <- build_schedule(pipeline_dir = pipeline_dir) 25 | 26 | schedule <- run_schedule( 27 | schedule, 28 | orch_frequency = "1 day", 29 | quiet = TRUE 30 | ) 31 | 32 | get_status(schedule) 33 | 34 | # Alternatively, use the underlying R6 method 35 | schedule$get_status() 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_parse_all_good/pipe1.R: -------------------------------------------------------------------------------- 1 | #' Simple mtcars print function 2 | #' 3 | #' This is a function that runs every hour starting at 4 | #' 2024-03-01 09:00:00 5 | #' 6 | #' @maestroFrequency 1 day 7 | #' @maestroStartTime 2024-03-01 09:00:00 8 | #' @maestroTz UTC 9 | #' @maestroLogLevel INFO 10 | #' 11 | #' @export 12 | get_mtcars <- function() { 13 | Sys.sleep(0.02) 14 | mtcars 15 | } 16 | 17 | #' Multiply 18 | #' 19 | #' @maestroFrequency 3 month 20 | #' @maestroLogLevel warn 21 | #' 22 | #' @export 23 | wait <- function() { 24 | Sys.sleep(0.01) 25 | } 26 | 27 | #' Add 28 | #' 29 | #' @maestroFrequency 3 month 30 | #' @maestroStartTime 1970-01-01 31 | #' 32 | #' @export 33 | add <- function() { 34 | invisible() 35 | } 36 | 37 | #' Something 38 | #' 39 | #' @maestroFrequency 3 month 40 | #' @maestroStartTime 1970-01-01 00:00:00 ADT 41 | #' 42 | #' @export 43 | something <- function() { 44 | invisible() 45 | } 46 | 47 | #' Daily 48 | #' 49 | #' @maestroFrequency daily 50 | #' @maestroStartTime 1970-01-01 00:00:00 ADT 51 | #' 52 | #' @export 53 | something2 <- function() { 54 | invisible() 55 | } 56 | -------------------------------------------------------------------------------- /man/get_schedule.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get_schedule.R 3 | \name{get_schedule} 4 | \alias{get_schedule} 5 | \title{Get the schedule from a MaestroSchedule object} 6 | \usage{ 7 | get_schedule(schedule) 8 | } 9 | \arguments{ 10 | \item{schedule}{object of type MaestroSchedule created using \code{build_schedule()}} 11 | } 12 | \value{ 13 | data.frame 14 | } 15 | \description{ 16 | A schedule is represented as a table where each row is a pipeline and 17 | the columns contain scheduling parameters such as the frequency and start time. 18 | } 19 | \details{ 20 | The schedule table is used internally in a MaestroSchedule object but can be 21 | accessed using this function or accessing the R6 method of the MaestroSchedule 22 | object. 23 | } 24 | \examples{ 25 | if (interactive()) { 26 | pipeline_dir <- tempdir() 27 | create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 28 | schedule <- build_schedule(pipeline_dir = pipeline_dir) 29 | 30 | get_schedule(schedule) 31 | 32 | # Alternatively, use the underlying R6 method 33 | schedule$get_schedule() 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # MIT License 2 | 3 | Copyright (c) 2024 maestro authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/dags.md: -------------------------------------------------------------------------------- 1 | # DAGs work as expected 2 | 3 | Code 4 | dag$get_network() 5 | Output 6 | # A tibble: 9 x 2 7 | from to 8 | 9 | 1 primary with_inputs 10 | 2 trunk branch1 11 | 3 trunk branch2 12 | 4 branch1 subbranch1 13 | 5 branch2 subbranch2 14 | 6 start high_road 15 | 7 start low_road 16 | 8 high_road end 17 | 9 low_road end 18 | 19 | # Even if a downstream pipeline is 'scheduled' it doesn't run unless the upstream component does 20 | 21 | Code 22 | schedule$get_status()$invoked 23 | Output 24 | [1] FALSE FALSE 25 | 26 | --- 27 | 28 | Code 29 | schedule$get_status()$invoked 30 | Output 31 | [1] FALSE FALSE FALSE 32 | 33 | # Even if a downstream pipeline is 'scheduled' it runs if the upstream component does 34 | 35 | Code 36 | schedule$get_status()$invoked 37 | Output 38 | [1] TRUE TRUE 39 | 40 | --- 41 | 42 | Code 43 | schedule$get_status()$invoked 44 | Output 45 | [1] TRUE TRUE TRUE 46 | 47 | -------------------------------------------------------------------------------- /tests/testthat/test-utils.R: -------------------------------------------------------------------------------- 1 | test_that("convert_to_seconds works", { 2 | 3 | res <- convert_to_seconds("1 week") 4 | expect_equal(res, unclass(lubridate::duration(1, units = "week"))) 5 | 6 | res <- convert_to_seconds("12 hours") 7 | expect_equal(res, unclass(lubridate::duration(12, units = "hours"))) 8 | 9 | res <- convert_to_seconds("25minutes") 10 | expect_equal(res, unclass(lubridate::duration(25, units = "minutes"))) 11 | 12 | res <- convert_to_seconds("48 days") 13 | expect_equal(res, unclass(lubridate::duration(48, units = "days"))) 14 | 15 | res <- convert_to_seconds("1 quarters") 16 | expect_equal(res, 7884000L) 17 | }) 18 | 19 | test_that("parse_rounding_unit works", { 20 | 21 | good_examples <- c( 22 | "1 days", "4 weeks", "10 months", "2 week", 23 | "15 minute", "30 day ", " 20 hours", "2 years", 24 | "2 quarters", "19 mins" 25 | ) 26 | 27 | purrr::walk(good_examples, ~{ 28 | expect_no_error( 29 | parse_rounding_unit(.x) 30 | ) 31 | }) 32 | 33 | bad_examples <- c( 34 | "one day", "45 daysasd", "3 hrs", "4" 35 | ) 36 | 37 | purrr::walk(bad_examples, ~{ 38 | expect_error( 39 | parse_rounding_unit(.x), 40 | regexp = "Invalid" 41 | ) 42 | }) 43 | }) 44 | -------------------------------------------------------------------------------- /R/get_artifacts.R: -------------------------------------------------------------------------------- 1 | #' Get the artifacts (return values) of the pipelines in a MaestroSchedule object. 2 | #' 3 | #' Artifacts are return values from pipelines. They are accessible as a named list 4 | #' where the names correspond to the names of the pipeline. 5 | #' @inheritParams run_schedule 6 | #' 7 | #' @return named list 8 | #' @export 9 | #' @examples 10 | #' if (interactive()) { 11 | #' pipeline_dir <- tempdir() 12 | #' create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 13 | #' schedule <- build_schedule(pipeline_dir = pipeline_dir) 14 | #' 15 | #' schedule <- run_schedule( 16 | #' schedule, 17 | #' orch_frequency = "1 day", 18 | #' quiet = TRUE 19 | #' ) 20 | #' 21 | #' get_artifacts(schedule) 22 | #' 23 | #' # Alternatively, use the underlying R6 method 24 | #' schedule$get_artifacts() 25 | #' } 26 | get_artifacts <- function(schedule) { 27 | 28 | if (!"MaestroSchedule" %in% class(schedule)) { 29 | cli::cli_abort( 30 | c("Schedule must be an object of {.cls MaestroSchedule} and not an object of class {.cls {class(schedule)}}.", 31 | "i" = "Use {.fn build_schedule} to create a valid schedule."), 32 | call = rlang::caller_env() 33 | ) 34 | } 35 | 36 | schedule$get_artifacts() 37 | } 38 | -------------------------------------------------------------------------------- /R/utils_cli.R: -------------------------------------------------------------------------------- 1 | #' cli output for generate schedule table 2 | #' 3 | #' @param parse_succeeds list of parse results (i.e., succeeded) 4 | #' @param parse_errors list of parse errors 5 | #' 6 | #' @keywords internal 7 | #' @return cli output 8 | maestro_parse_cli <- function(parse_succeeds, parse_errors) { 9 | 10 | n_fails <- length(parse_errors) 11 | n_succeeds <- length(parse_succeeds) 12 | 13 | if(n_succeeds == 0) { 14 | 15 | cli::cli_abort( 16 | c( 17 | "x" = "All scripts failed to parse", 18 | "i" = "See full error output with {.fn last_build_errors}" 19 | ), 20 | call = rlang::caller_env() 21 | ) 22 | } else { 23 | 24 | if (n_succeeds > 0) { 25 | cli::cli_inform( 26 | c("i" = "{n_succeeds} script{?s} successfully parsed") 27 | ) 28 | } 29 | 30 | if (n_fails > 0) { 31 | fail_vec <- purrr::map_chr(parse_errors, ~{ 32 | .x$message 33 | }) |> 34 | stats::setNames("!") 35 | 36 | cli::cli_warn( 37 | c( 38 | "{n_fails} script{?s} failed to parse:", 39 | fail_vec, 40 | "i" = "See full error output with {.fn last_build_errors}" 41 | ) 42 | ) 43 | } 44 | } 45 | 46 | return(invisible()) 47 | } 48 | -------------------------------------------------------------------------------- /R/get_status.R: -------------------------------------------------------------------------------- 1 | #' Get the statuses of the pipelines in a MaestroSchedule object 2 | #' 3 | #' A status data.frame contains the names and locations of the pipelines as 4 | #' well as information around whether they were invoked, the status (error, warning, etc.), 5 | #' and the run time. 6 | #' 7 | #' @inheritParams run_schedule 8 | #' 9 | #' @return data.frame 10 | #' @export 11 | #' @examples 12 | #' if (interactive()) { 13 | #' pipeline_dir <- tempdir() 14 | #' create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 15 | #' schedule <- build_schedule(pipeline_dir = pipeline_dir) 16 | #' 17 | #' schedule <- run_schedule( 18 | #' schedule, 19 | #' orch_frequency = "1 day", 20 | #' quiet = TRUE 21 | #' ) 22 | #' 23 | #' get_status(schedule) 24 | #' 25 | #' # Alternatively, use the underlying R6 method 26 | #' schedule$get_status() 27 | #' } 28 | get_status <- function(schedule) { 29 | 30 | if (!"MaestroSchedule" %in% class(schedule)) { 31 | cli::cli_abort( 32 | c("Schedule must be an object of {.cls MaestroSchedule} and not an object of class {.cls {class(schedule)}}.", 33 | "i" = "Use {.fn build_schedule} to create a valid schedule."), 34 | call = rlang::caller_env() 35 | ) 36 | } 37 | 38 | schedule$get_status() 39 | } 40 | -------------------------------------------------------------------------------- /R/get_schedule.R: -------------------------------------------------------------------------------- 1 | #' Get the schedule from a MaestroSchedule object 2 | #' 3 | #' A schedule is represented as a table where each row is a pipeline and 4 | #' the columns contain scheduling parameters such as the frequency and start time. 5 | #' 6 | #' The schedule table is used internally in a MaestroSchedule object but can be 7 | #' accessed using this function or accessing the R6 method of the MaestroSchedule 8 | #' object. 9 | #' 10 | #' @inheritParams run_schedule 11 | #' 12 | #' @return data.frame 13 | #' @export 14 | #' @examples 15 | #' if (interactive()) { 16 | #' pipeline_dir <- tempdir() 17 | #' create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 18 | #' schedule <- build_schedule(pipeline_dir = pipeline_dir) 19 | #' 20 | #' get_schedule(schedule) 21 | #' 22 | #' # Alternatively, use the underlying R6 method 23 | #' schedule$get_schedule() 24 | #' } 25 | get_schedule <- function(schedule) { 26 | 27 | if (!"MaestroSchedule" %in% class(schedule)) { 28 | cli::cli_abort( 29 | c("Schedule must be an object of {.cls MaestroSchedule} and not an object of class {.cls {class(schedule)}}.", 30 | "i" = "Use {.fn build_schedule} to create a valid schedule."), 31 | call = rlang::caller_env() 32 | ) 33 | } 34 | 35 | schedule$get_schedule() 36 | } 37 | -------------------------------------------------------------------------------- /man/show_network.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/show_network.R 3 | \name{show_network} 4 | \alias{show_network} 5 | \title{Visualize the schedule as a DAG} 6 | \usage{ 7 | show_network(schedule) 8 | } 9 | \arguments{ 10 | \item{schedule}{object of type MaestroSchedule created using \code{build_schedule()}} 11 | } 12 | \value{ 13 | DiagrammeR visualization 14 | } 15 | \description{ 16 | Create an interactive network visualization to show the dependency structure 17 | of pipelines in the schedule. This is only useful if there are pipelines in 18 | the schedule that take inputs/outputs from other pipelines. 19 | } 20 | \details{ 21 | Note that running this function on a schedule with all independent pipelines 22 | will produce a network visual with no connections. 23 | 24 | This function requires the installation of \code{DiagrammeR} which is not automatically 25 | installed with \code{maestro}. 26 | } 27 | \examples{ 28 | if (interactive()) { 29 | pipeline_dir <- tempdir() 30 | create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 31 | schedule <- build_schedule(pipeline_dir = pipeline_dir) 32 | 33 | schedule <- run_schedule( 34 | schedule, 35 | orch_frequency = "1 day", 36 | quiet = TRUE 37 | ) 38 | 39 | show_network(schedule) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /man/figures/README-/schedule-dark.svg: -------------------------------------------------------------------------------- 1 | ℹ3pipelinessuccessfullyparsed -------------------------------------------------------------------------------- /man/figures/README-/schedule.svg: -------------------------------------------------------------------------------- 1 | ℹ3pipelinessuccessfullyparsed -------------------------------------------------------------------------------- /tests/testthat/test-get_flags.R: -------------------------------------------------------------------------------- 1 | test_that("get_flags works as expected", { 2 | withr::with_tempdir({ 3 | dir.create("pipelines") 4 | writeLines( 5 | " 6 | #' @maestroFrequency hourly 7 | #' @maestroFlags experimental dev 8 | p1 <- function() { 9 | } 10 | 11 | #' @maestroFrequency 2 hours 12 | #' @maestroFlags critical aviation 13 | p2 <- function() { 14 | } 15 | 16 | #' @maestroFrequency 3 hours 17 | #' @maestroFlags critical 18 | p3 <- function() { 19 | } 20 | ", 21 | con = "pipelines/flags.R" 22 | ) 23 | 24 | schedule <- build_schedule(quiet = TRUE) 25 | }) 26 | 27 | flags_df <- get_flags(schedule) 28 | 29 | expect_snapshot(flags_df) 30 | }) 31 | 32 | test_that("returns empty data.frame if there are no flags", { 33 | withr::with_tempdir({ 34 | dir.create("pipelines") 35 | writeLines( 36 | " 37 | #' @maestroFrequency hourly 38 | p1 <- function() { 39 | } 40 | 41 | #' @maestroFrequency 2 hours 42 | p2 <- function() { 43 | } 44 | 45 | #' @maestroFrequency 3 hours 46 | p3 <- function() { 47 | } 48 | ", 49 | con = "pipelines/flags.R" 50 | ) 51 | 52 | schedule <- build_schedule(quiet = TRUE) 53 | }) 54 | 55 | flags_df <- get_flags(schedule) 56 | 57 | expect_snapshot(flags_df) 58 | }) 59 | -------------------------------------------------------------------------------- /man/suggest_orch_frequency.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/suggest_orch_frequency.R 3 | \name{suggest_orch_frequency} 4 | \alias{suggest_orch_frequency} 5 | \title{Suggest orchestrator frequency based on a schedule} 6 | \usage{ 7 | suggest_orch_frequency( 8 | schedule, 9 | check_datetime = lubridate::now(tzone = "UTC") 10 | ) 11 | } 12 | \arguments{ 13 | \item{schedule}{MaestroSchedule object created by \code{build_schedule()}} 14 | 15 | \item{check_datetime}{datetime against which to check the running of pipelines (default is current system time in UTC)} 16 | } 17 | \value{ 18 | frequency string 19 | } 20 | \description{ 21 | Suggests a frequency to run the orchestrator based on the frequencies of the 22 | pipelines in a schedule. 23 | } 24 | \details{ 25 | This function attempts to find the smallest interval of time between all pipelines. 26 | If the smallest interval is less than 15 minutes, it just uses the smallest interval. 27 | 28 | Note this function is intended to be used interactively when deciding how often to 29 | schedule the orchestrator. Programmatic use is not recommended. 30 | } 31 | \examples{ 32 | 33 | if (interactive()) { 34 | pipeline_dir <- tempdir() 35 | create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 36 | schedule <- build_schedule(pipeline_dir = pipeline_dir) 37 | suggest_orch_frequency(schedule) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /man/build_schedule.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/build_schedule.R 3 | \name{build_schedule} 4 | \alias{build_schedule} 5 | \title{Build a schedule table} 6 | \usage{ 7 | build_schedule(pipeline_dir = "./pipelines", quiet = FALSE) 8 | } 9 | \arguments{ 10 | \item{pipeline_dir}{path to directory containing the pipeline scripts} 11 | 12 | \item{quiet}{silence metrics to the console (default = \code{FALSE})} 13 | } 14 | \value{ 15 | MaestroSchedule 16 | } 17 | \description{ 18 | Builds a schedule data.frame for scheduling pipelines in \code{run_schedule()}. 19 | } 20 | \details{ 21 | This function parses the maestro tags of functions located in \code{pipeline_dir} which is 22 | conventionally called 'pipelines'. An orchestrator requires a schedule table 23 | to determine which pipelines are to run and when. Each row in a schedule table 24 | is a pipeline name and its scheduling parameters such as its frequency. 25 | 26 | The schedule table is mostly intended to be used by \code{run_schedule()} immediately. 27 | In other words, it is not recommended to make changes to it. 28 | } 29 | \examples{ 30 | 31 | # Creating a temporary directory for demo purposes! In practice, just 32 | # create a 'pipelines' directory at the project level. 33 | 34 | if (interactive()) { 35 | pipeline_dir <- tempdir() 36 | create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 37 | build_schedule(pipeline_dir = pipeline_dir) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /R/show_network.R: -------------------------------------------------------------------------------- 1 | #' Visualize the schedule as a DAG 2 | #' 3 | #' Create an interactive network visualization to show the dependency structure 4 | #' of pipelines in the schedule. This is only useful if there are pipelines in 5 | #' the schedule that take inputs/outputs from other pipelines. 6 | #' 7 | #' Note that running this function on a schedule with all independent pipelines 8 | #' will produce a network visual with no connections. 9 | #' 10 | #' This function requires the installation of `DiagrammeR` which is not automatically 11 | #' installed with `maestro`. 12 | #' 13 | #' @inheritParams run_schedule 14 | #' 15 | #' @return DiagrammeR visualization 16 | #' @export 17 | #' 18 | #' @examples 19 | #' if (interactive()) { 20 | #' pipeline_dir <- tempdir() 21 | #' create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 22 | #' schedule <- build_schedule(pipeline_dir = pipeline_dir) 23 | #' 24 | #' schedule <- run_schedule( 25 | #' schedule, 26 | #' orch_frequency = "1 day", 27 | #' quiet = TRUE 28 | #' ) 29 | #' 30 | #' show_network(schedule) 31 | #' } 32 | show_network <- function(schedule) { 33 | 34 | if (!"MaestroSchedule" %in% class(schedule)) { 35 | cli::cli_abort( 36 | c("Schedule must be an object of {.cls MaestroSchedule} and not an object of class {.cls {class(schedule)}}.", 37 | "i" = "Use {.fn build_schedule} to create a valid schedule."), 38 | call = rlang::caller_env() 39 | ) 40 | } 41 | 42 | schedule$show_network() 43 | } 44 | -------------------------------------------------------------------------------- /man/get_pipeline_run_sequence.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{get_pipeline_run_sequence} 4 | \alias{get_pipeline_run_sequence} 5 | \title{Generate a sequence of run times for a pipeline} 6 | \usage{ 7 | get_pipeline_run_sequence( 8 | pipeline_n, 9 | pipeline_unit, 10 | pipeline_datetime, 11 | check_datetime, 12 | pipeline_hours = 0:23, 13 | pipeline_days_of_week = 1:7, 14 | pipeline_days_of_month = 1:31, 15 | pipeline_months = 1:12 16 | ) 17 | } 18 | \arguments{ 19 | \item{pipeline_n}{number of units for the pipeline frequency} 20 | 21 | \item{pipeline_unit}{unit for the pipeline frequency} 22 | 23 | \item{pipeline_datetime}{datetime of the first time the pipeline is to run} 24 | 25 | \item{check_datetime}{datetime against which to check the running of pipelines (default is current system time in UTC)} 26 | 27 | \item{pipeline_hours}{vector of integers [0-23] corresponding to hours of day for the pipeline to run} 28 | 29 | \item{pipeline_days_of_week}{vector of integers [1-7] corresponding to days of week for the pipeline to run (1 = Sunday)} 30 | 31 | \item{pipeline_days_of_month}{vector of integers [1-31] corresponding to days of month for the pipeline to run} 32 | 33 | \item{pipeline_months}{vector of integers [1-12] corresponding to months of year for the pipeline to run} 34 | } 35 | \value{ 36 | vector of timestamps or dates 37 | } 38 | \description{ 39 | Generate a sequence of run times for a pipeline 40 | } 41 | \keyword{internal} 42 | -------------------------------------------------------------------------------- /inst/roxygen2-tags.yml: -------------------------------------------------------------------------------- 1 | - name: maestroFrequency 2 | description: > 3 | The base unit for scheduling a pipeline. Pipeline will run at the 4 | frequency defined here. Default is 1 day. 5 | 6 | Other examples are 3 weeks, 12 hours, 1 quarter, 2 months, 1 year. 7 | template: ' ${1:maestroFrequency}' 8 | recommend: true 9 | 10 | - name: maestroStartTime 11 | description: > 12 | Initial time of the pipeline run. This also defines on what time pipeline will recur. 13 | Must be in yyyy-MM-dd HH:MM:SS format. Default is 1970-01-01 00:00:00. 14 | 15 | For example, if pipeline has a frequency of 'day' and an interval of 1 and the 16 | start time is 2024-01-01 12:00:00, then it will run daily on the 12:00 hour. 17 | template: ' ${1:maestroStartTime}' 18 | recommend: true 19 | 20 | - name: maestroTz 21 | description: > 22 | Timezone of the start time. Supported time zones are found using `r OlsonNames()`. 23 | Default is UTC. 24 | template: ' ${1:maestroTz}' 25 | recommend: true 26 | 27 | - name: maestroSkip 28 | description: > 29 | Skip a pipeline during orchestration if tag is present. 30 | 31 | Useful during development. Skipped pipelines are indicated in console output of 32 | `r maestro::run_schedule()` 33 | template: ' ${1:maestroSkip}' 34 | recommend: true 35 | 36 | - name: maestroLogLevel 37 | description: > 38 | Level of logging threshold if using logging. 39 | 40 | Available values include INFO, WARN, and ERROR. 41 | template: ' ${1:maestroLogLevel}' 42 | recommend: true 43 | -------------------------------------------------------------------------------- /tests/testthat/test_pipelines_dags_good/dags.R: -------------------------------------------------------------------------------- 1 | # Simple 1 -> 1 ----------------------------------------------------------- 2 | #' @maestroInputs primary 3 | with_inputs <- function(.input) { 4 | paste("input message is:", .input) 5 | } 6 | 7 | #' @maestroFrequency daily 8 | #' @maestroOutputs with_inputs 9 | primary <- function() { 10 | "hello" 11 | } 12 | 13 | 14 | # Tree with Two Branches x 2 ---------------------------------------------- 15 | #' @maestroFrequency daily 16 | #' @maestroOutputs branch1 branch2 17 | trunk <- function() { 18 | 1 19 | } 20 | 21 | #' @maestroInputs trunk 22 | #' @maestroOutputs subbranch1 23 | branch1 <- function(.input) { 24 | .input * 1 25 | } 26 | 27 | #' @maestroInputs trunk 28 | #' @maestroOutputs subbranch2 29 | branch2 <- function(.input) { 30 | .input * 2 31 | } 32 | 33 | #' @maestroInputs branch1 34 | subbranch1 <- function(.input) { 35 | .input * 2 36 | } 37 | 38 | #' @maestroInputs branch2 39 | subbranch2 <- function(.input) { 40 | .input * 3 41 | } 42 | 43 | # Split and Unite --------------------------------------------------------- 44 | 45 | #' @maestroOutputs high_road low_road 46 | start <- function() { 47 | c("a", "A") 48 | } 49 | 50 | #' @maestroInputs start 51 | #' @maestroOutputs end 52 | high_road <- function(.input) { 53 | toupper(.input) 54 | } 55 | 56 | #' @maestroInputs start 57 | #' @maestroOutputs end 58 | low_road <- function(.input) { 59 | tolower(.input) 60 | } 61 | 62 | #' @maestroInputs high_road low_road 63 | end <- function(.input) { 64 | c(.input, "b") 65 | } 66 | 67 | 68 | -------------------------------------------------------------------------------- /.github/workflows/pkgdown.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | branches: [main, master] 8 | release: 9 | types: [published] 10 | workflow_dispatch: 11 | 12 | name: pkgdown.yaml 13 | 14 | permissions: read-all 15 | 16 | jobs: 17 | pkgdown: 18 | runs-on: ubuntu-latest 19 | # Only restrict concurrency for non-PR jobs 20 | concurrency: 21 | group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} 22 | env: 23 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 24 | permissions: 25 | contents: write 26 | steps: 27 | - uses: actions/checkout@v4 28 | 29 | - uses: r-lib/actions/setup-pandoc@v2 30 | 31 | - uses: r-lib/actions/setup-r@v2 32 | with: 33 | use-public-rspm: true 34 | 35 | - uses: r-lib/actions/setup-r-dependencies@v2 36 | with: 37 | extra-packages: any::pkgdown, local::. 38 | needs: website 39 | 40 | - name: Build site 41 | run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) 42 | shell: Rscript {0} 43 | 44 | - name: Deploy to GitHub pages 🚀 45 | if: github.event_name != 'pull_request' 46 | uses: JamesIves/github-pages-deploy-action@v4.5.0 47 | with: 48 | clean: false 49 | branch: gh-pages 50 | folder: docs 51 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: maestro 2 | Type: Package 3 | Title: Orchestration of Data Pipelines 4 | Version: 0.7.1 5 | Authors@R: 6 | c( 7 | person("Will", "Hipson", , "will.e.hipson@gmail.com", role = c("cre", "aut", "cph"), 8 | comment = c(ORCID = "0000-0002-3931-2189")), 9 | person("Ryan", "Garnett", , "ryangarnett78@gmail.com", role = c("aut", "ctb", "cph")) 10 | ) 11 | Maintainer: Will Hipson 12 | Description: Framework for creating and orchestrating data pipelines. Organize, orchestrate, and monitor multiple pipelines in a single project. Use tags to decorate functions with scheduling parameters and configuration. 13 | License: MIT + file LICENSE 14 | URL: https://github.com/whipson/maestro, https://whipson.github.io/maestro/ 15 | BugReports: https://github.com/whipson/maestro/issues 16 | Encoding: UTF-8 17 | LazyData: true 18 | Imports: 19 | cli (>= 3.3.0), 20 | dplyr (>= 1.1.0), 21 | glue, 22 | lifecycle, 23 | logger, 24 | lubridate (>= 1.9.1), 25 | purrr (>= 1.0.0), 26 | R.utils, 27 | R6, 28 | rlang (>= 1.0.0), 29 | roxygen2, 30 | tictoc, 31 | timechange, 32 | utils 33 | Roxygen: list(markdown = TRUE, roclets = c("rd", "collate", "namespace")) 34 | RoxygenNote: 7.3.3 35 | Depends: 36 | R (>= 4.1.0) 37 | Suggests: 38 | asciicast, 39 | DiagrammeR, 40 | furrr, 41 | future, 42 | knitr, 43 | quarto, 44 | rmarkdown, 45 | rstudioapi, 46 | testthat (>= 3.0.0), 47 | withr 48 | Config/testthat/edition: 3 49 | VignetteBuilder: 50 | knitr, 51 | quarto 52 | -------------------------------------------------------------------------------- /.github/workflows/R-CMD-check.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | 8 | name: R-CMD-check.yaml 9 | 10 | permissions: read-all 11 | 12 | jobs: 13 | R-CMD-check: 14 | runs-on: ${{ matrix.config.os }} 15 | 16 | name: ${{ matrix.config.os }} (${{ matrix.config.r }}) 17 | 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | config: 22 | - {os: macos-latest, r: 'release'} 23 | - {os: windows-latest, r: 'release'} 24 | - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} 25 | - {os: ubuntu-latest, r: 'release'} 26 | - {os: ubuntu-latest, r: 'oldrel-1'} 27 | 28 | env: 29 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 30 | R_KEEP_PKG_SOURCE: yes 31 | 32 | steps: 33 | - uses: actions/checkout@v4 34 | 35 | - uses: r-lib/actions/setup-pandoc@v2 36 | 37 | - uses: r-lib/actions/setup-r@v2 38 | with: 39 | r-version: ${{ matrix.config.r }} 40 | http-user-agent: ${{ matrix.config.http-user-agent }} 41 | use-public-rspm: true 42 | 43 | - uses: r-lib/actions/setup-r-dependencies@v2 44 | with: 45 | extra-packages: any::rcmdcheck 46 | needs: check 47 | 48 | - uses: r-lib/actions/check-r-package@v2 49 | with: 50 | upload-snapshots: true 51 | build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")' 52 | -------------------------------------------------------------------------------- /tests/testthat/test-build_schedule.R: -------------------------------------------------------------------------------- 1 | test_that("build_schedule works on a directory of all good pipelines", { 2 | res <- build_schedule(test_path("test_pipelines_parse_all_good")) 3 | expect_s3_class(res, "MaestroSchedule") 4 | schedule <- res$get_schedule() 5 | expect_s3_class(schedule, "data.frame") 6 | }) |> 7 | suppressMessages() 8 | 9 | test_that("build_schedule works on a directory of some good pipelines, warns", { 10 | 11 | expect_warning({ 12 | res <- build_schedule(test_path("test_pipelines_parse_some_good")) 13 | }, regexp = "failed to parse") 14 | 15 | expect_s3_class(res, "MaestroSchedule") 16 | expect_gte(length(res$PipelineList), 1) 17 | }) |> 18 | suppressMessages() 19 | 20 | test_that("build_schedule errors on a directory of all bad pipelines", { 21 | expect_error({ 22 | res <- build_schedule(test_path("test_pipelines_parse_all_bad")) 23 | }, regexp = "All scripts failed to parse") 24 | 25 | errors <- last_build_errors() 26 | expect_type(errors, "list") 27 | expect_gt(length(errors), 0) 28 | }) |> 29 | suppressMessages() 30 | 31 | test_that("build_schedule errors on a nonexistent directory with no .R scripts", { 32 | expect_error({ 33 | build_schedule(test_path("directory_that_doesnt_exist")) 34 | }, regexp = "No directory called") 35 | }) 36 | 37 | test_that("informs if referencing a folder with no R scripts", { 38 | withr::with_tempdir({ 39 | expect_message({ 40 | build_schedule(tempdir()) 41 | }, regexp = "No R scripts in") 42 | }) 43 | }) 44 | 45 | test_that("errors if there are duplicate pipeline names", { 46 | expect_error({ 47 | build_schedule(test_path("test_pipelines_dup_names"), quiet = TRUE) 48 | }, "Function names must all be unique") 49 | }) 50 | -------------------------------------------------------------------------------- /R/create_maestro.R: -------------------------------------------------------------------------------- 1 | #' Creates a new maestro project 2 | #' 3 | #' @inheritParams create_orchestrator 4 | #' @param ... unused 5 | #' 6 | #' @export 7 | #' @return invisible 8 | #' @examples 9 | #' 10 | #' # Creates a new maestro project with an R orchestrator 11 | #' if (interactive()) { 12 | #' new_proj_dir <- tempdir() 13 | #' create_maestro(new_proj_dir, type = "R", overwrite = TRUE) 14 | #' 15 | #' create_maestro(new_proj_dir, type = "Quarto", overwrite = TRUE) 16 | #' } 17 | create_maestro <- function(path, type = "R", overwrite = FALSE, quiet = FALSE, ...) { 18 | 19 | type <- match.arg(type, choices = c("R", "Quarto", "RMarkdown")) 20 | 21 | path_to_maestro <- normalizePath( 22 | path, 23 | mustWork = FALSE 24 | ) 25 | 26 | if (dir.exists(path_to_maestro)) { 27 | if (!overwrite) { 28 | cli::cli_abort( 29 | paste( 30 | "Project directory already exists. \n", 31 | "Set `create_maestro(overwrite = TRUE)` to overwrite anyway.\n", 32 | "This will remove any work in this directory. \n" 33 | ), 34 | call = NULL 35 | ) 36 | } else { 37 | if (!quiet) cli::cli_alert_warning("Overwriting existing project.") 38 | } 39 | } 40 | 41 | if (!quiet) cli::cli_alert_success("Creating maestro project") 42 | 43 | # ensure path exists 44 | dir.create(path, recursive = TRUE, showWarnings = FALSE) 45 | 46 | create_orchestrator( 47 | path = file.path(path, "orchestrator"), 48 | type = type, 49 | open = FALSE, 50 | quiet = TRUE, 51 | overwrite = overwrite 52 | ) 53 | 54 | create_pipeline( 55 | pipe_name = "my_pipe", 56 | pipeline_dir = file.path(path, "pipelines"), 57 | open = FALSE, 58 | quiet = TRUE, 59 | overwrite = overwrite 60 | ) 61 | } 62 | -------------------------------------------------------------------------------- /man/invoke.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/invoke.R 3 | \name{invoke} 4 | \alias{invoke} 5 | \title{Manually run a pipeline regardless of schedule} 6 | \usage{ 7 | invoke( 8 | schedule, 9 | pipe_name, 10 | resources = list(), 11 | ..., 12 | quiet = TRUE, 13 | log_to_console = FALSE 14 | ) 15 | } 16 | \arguments{ 17 | \item{schedule}{object of type MaestroSchedule created using \code{build_schedule()}} 18 | 19 | \item{pipe_name}{name of a single pipe name from the schedule} 20 | 21 | \item{resources}{named list of shared resources made available to pipelines as needed} 22 | 23 | \item{...}{other arguments passed to \code{run_schedule()}} 24 | 25 | \item{quiet}{silence metrics to the console (default = \code{FALSE}). Note this does not affect messages generated from pipelines when \code{log_to_console = TRUE}.} 26 | 27 | \item{log_to_console}{whether or not to include pipeline messages, warnings, errors to the console (default = \code{FALSE}) (see Logging & Console Output section)} 28 | } 29 | \value{ 30 | invisible 31 | } 32 | \description{ 33 | Instantly run a single pipeline from the schedule. This is useful for testing 34 | purposes or if you want to just run something one-off. 35 | } 36 | \details{ 37 | Scheduling parameters such as the frequency, start time, and specifiers are ignored. 38 | The pipeline will be run even if \code{maestroSkip} is present. If the pipeline is a DAG 39 | pipeline, \code{invoke} will attempt to execute the full DAG. 40 | } 41 | \examples{ 42 | if (interactive()) { 43 | pipeline_dir <- tempdir() 44 | create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 45 | schedule <- build_schedule(pipeline_dir = pipeline_dir) 46 | 47 | invoke(schedule, "my_new_pipeline") 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /R/latest_getters.R: -------------------------------------------------------------------------------- 1 | #' Retrieve latest maestro build errors 2 | #' 3 | #' Gets the latest schedule build errors following use of `build_schedule()`. If 4 | #' the build succeeded or `build_schedule()` has not been run it will be `NULL`. 5 | #' 6 | #' @return error messages 7 | #' @export 8 | #' @examples 9 | #' 10 | #' last_build_errors() 11 | last_build_errors <- function() { 12 | maestro_pkgenv$last_build_errors 13 | } 14 | 15 | #' Retrieve latest maestro pipeline errors 16 | #' 17 | #' Gets the latest pipeline errors following use of `run_schedule()`. If 18 | #' the all runs succeeded or `run_schedule()` has not been run it will be `NULL`. 19 | #' 20 | #' @return error messages 21 | #' @export 22 | #' @examples 23 | #' 24 | #' last_run_errors() 25 | last_run_errors <- function() { 26 | maestro_pkgenv$last_run_errors 27 | } 28 | 29 | #' Retrieve latest maestro pipeline warnings 30 | #' 31 | #' Gets the latest pipeline warnings following use of `run_schedule()`. If 32 | #' there are no warnings or `run_schedule()` has not been run it will be `NULL`. 33 | #' 34 | #' Note that setting `maestroLogLevel` to something greater than `WARN` will 35 | #' ignore warnings. 36 | #' 37 | #' @return warning messages 38 | #' @export 39 | #' @examples 40 | #' 41 | #' last_run_warnings() 42 | last_run_warnings <- function() { 43 | maestro_pkgenv$last_run_warnings 44 | } 45 | 46 | #' Retrieve latest maestro pipeline messages 47 | #' 48 | #' Gets the latest pipeline messages following use of `run_schedule()`. If 49 | #' there are no messages or `run_schedule()` has not been run it will be `NULL`. 50 | #' 51 | #' Note that setting `maestroLogLevel` to something greater than `INFO` will 52 | #' ignore messages. 53 | #' 54 | #' @return messages 55 | #' @export 56 | #' @examples 57 | #' 58 | #' last_run_messages() 59 | last_run_messages <- function() { 60 | maestro_pkgenv$last_run_messages 61 | } 62 | -------------------------------------------------------------------------------- /R/create_orchestrator.R: -------------------------------------------------------------------------------- 1 | #' Create a new orchestrator 2 | #' 3 | #' @param open whether or not to open the script upon creation 4 | #' @param path file path for the orchestrator script 5 | #' @param type file type for the orchestrator (supports R, Quarto, and RMarkdown) 6 | #' @param quiet whether to silence messages in the console (default = `FALSE`) 7 | #' @param overwrite whether to overwrite an existing orchestrator or maestro project 8 | #' 9 | #' @return invisible 10 | create_orchestrator <- function( 11 | path, 12 | type = c("R", "Quarto", "RMarkdown"), 13 | open = interactive(), 14 | quiet = FALSE, 15 | overwrite = FALSE 16 | ) { 17 | 18 | type <- match.arg(type, choices = c("R", "Quarto", "RMarkdown")) 19 | 20 | template <- ifelse(type == "R", "orchestrator_template", "orchestrator_template_qmd") 21 | 22 | script <- readLines(system.file(template, package = "maestro")) |> 23 | paste(collapse = "\n") |> 24 | glue::glue( 25 | .open = "{{", 26 | .close = "}}", 27 | .null = NULL 28 | ) 29 | 30 | extension <- switch (type, 31 | R = "R", 32 | Quarto = "qmd", 33 | RMarkdown = "Rmd" 34 | ) 35 | 36 | path <- paste0(path, ".", extension) 37 | 38 | if (file.exists(path)) { 39 | if (!overwrite) { 40 | cli::cli_abort( 41 | c("File {.file {path}} already exists.", 42 | "Set {.code maestro::create_orchestrator(overwrite = TRUE)} to overwrite anyway."), 43 | call = NULL 44 | ) 45 | } else { 46 | if (!quiet) cli::cli_alert_warning("Overwriting existing orchestrator at {.file {path}}.") 47 | } 48 | } 49 | 50 | writeLines( 51 | script, 52 | path 53 | ) 54 | 55 | if (open) { 56 | rlang::check_installed("rstudioapi") 57 | rstudioapi::documentOpen(path) 58 | } 59 | 60 | if(!quiet) { 61 | cli::cli_alert_success("Created orchestrator at {.file {path}}") 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /.github/workflows/test-coverage.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | branches: [main, master] 8 | 9 | name: test-coverage 10 | 11 | permissions: read-all 12 | 13 | jobs: 14 | test-coverage: 15 | runs-on: ubuntu-latest 16 | env: 17 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | 22 | - uses: r-lib/actions/setup-r@v2 23 | with: 24 | use-public-rspm: true 25 | 26 | - uses: r-lib/actions/setup-r-dependencies@v2 27 | with: 28 | extra-packages: any::covr, any::xml2 29 | needs: coverage 30 | 31 | - name: Test coverage 32 | run: | 33 | cov <- covr::package_coverage( 34 | quiet = FALSE, 35 | clean = FALSE, 36 | install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package") 37 | ) 38 | covr::to_cobertura(cov) 39 | shell: Rscript {0} 40 | 41 | - uses: codecov/codecov-action@v4 42 | with: 43 | fail_ci_if_error: ${{ github.event_name != 'pull_request' && true || false }} 44 | file: ./cobertura.xml 45 | plugin: noop 46 | disable_search: true 47 | token: ${{ secrets.CODECOV_TOKEN }} 48 | 49 | - name: Show testthat output 50 | if: always() 51 | run: | 52 | ## -------------------------------------------------------------------- 53 | find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true 54 | shell: bash 55 | 56 | - name: Upload test results 57 | if: failure() 58 | uses: actions/upload-artifact@v4 59 | with: 60 | name: coverage-test-failures 61 | path: ${{ runner.temp }}/package 62 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/conditionals.md: -------------------------------------------------------------------------------- 1 | # Conditional pipes work with DAG pipelines via .input 2 | 3 | Code 4 | status$invoked 5 | Output 6 | [1] TRUE TRUE 7 | 8 | --- 9 | 10 | Code 11 | status$invoked 12 | Output 13 | [1] TRUE FALSE 14 | 15 | --- 16 | 17 | Code 18 | status$invoked 19 | Output 20 | [1] TRUE FALSE 21 | 22 | # DAGs with a conditional pipe in the middle by default halt further execution of the DAG 23 | 24 | Code 25 | status$invoked 26 | Output 27 | [1] TRUE FALSE FALSE 28 | 29 | # Conditional pipes work using resources 30 | 31 | Code 32 | status$invoked 33 | Output 34 | [1] TRUE FALSE 35 | 36 | # Conditions with errors are handled 37 | 38 | Code 39 | status$success 40 | Output 41 | [1] FALSE 42 | 43 | --- 44 | 45 | Code 46 | status$invoked 47 | Output 48 | [1] TRUE 49 | 50 | --- 51 | 52 | Code 53 | last_run_errors() 54 | Output 55 | $p1 56 | 57 | 58 | 59 | # Conditions that don't return a single boolean are handled 60 | 61 | Code 62 | status$success 63 | Output 64 | [1] FALSE 65 | 66 | --- 67 | 68 | Code 69 | status$invoked 70 | Output 71 | [1] TRUE 72 | 73 | --- 74 | 75 | Code 76 | last_run_errors() 77 | Output 78 | $p1 79 | 80 | 81 | 82 | # Empty maestroRunIf is ignored 83 | 84 | Code 85 | status$success 86 | Output 87 | [1] TRUE 88 | 89 | --- 90 | 91 | Code 92 | status$invoked 93 | Output 94 | [1] TRUE 95 | 96 | # Branching pipelines execute with conditionals 97 | 98 | Code 99 | status$invoked 100 | Output 101 | [1] TRUE TRUE FALSE 102 | 103 | -------------------------------------------------------------------------------- /man/get_slot_usage.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get_slot_usage.R 3 | \name{get_slot_usage} 4 | \alias{get_slot_usage} 5 | \title{Get time slot usage of a schedule} 6 | \usage{ 7 | get_slot_usage(schedule, orch_frequency, slot_interval = "hour") 8 | } 9 | \arguments{ 10 | \item{schedule}{object of type MaestroSchedule created using \code{build_schedule()}} 11 | 12 | \item{orch_frequency}{of the orchestrator, a single string formatted like "1 day", "2 weeks", "hourly", etc.} 13 | 14 | \item{slot_interval}{a time unit indicating the interval of time to consider between slots (e.g., 'hour', 'day')} 15 | } 16 | \value{ 17 | data.frame 18 | } 19 | \description{ 20 | Get the number of pipelines scheduled to run for each time slot at a particular 21 | slot interval. Time slots are times that the orchestrator runs and the slot interval 22 | determines the level of granularity to consider. 23 | } 24 | \details{ 25 | This function is particularly useful when you have multiple pipelines in a project 26 | and you want to see what recurring time intervals may be available or underused 27 | for new pipelines. 28 | 29 | Note that this function is intended for use in an interactive session while developing 30 | a maestro project. It is not intended for use in the orchestrator. 31 | 32 | As an example, consider we have four pipelines running at various frequencies 33 | and the orchestrator running every hour. Then let's say there's to be a new 34 | pipeline that runs every day. One might ask 'what hour should I schedule this new 35 | pipeline to run on?'. By using \code{get_slot_usage(schedule, orch_frequency = '1 hour', slot_interval = 'hour')} 36 | on the existing schedule, you could identify for each hour how many pipelines 37 | are already scheduled to run and choose the ones with the lowest usage. 38 | } 39 | \examples{ 40 | if (interactive()) { 41 | pipeline_dir <- tempdir() 42 | create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 43 | schedule <- build_schedule(pipeline_dir = pipeline_dir) 44 | 45 | get_slot_usage( 46 | schedule, 47 | orch_frequency = "1 hour", 48 | slot_interval = "hour" 49 | ) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | S3method(roclet_process,roclet_maestro) 4 | S3method(roclet_process,roclet_maestroDays) 5 | S3method(roclet_process,roclet_maestroFlags) 6 | S3method(roclet_process,roclet_maestroFrequency) 7 | S3method(roclet_process,roclet_maestroHours) 8 | S3method(roclet_process,roclet_maestroInputs) 9 | S3method(roclet_process,roclet_maestroLabel) 10 | S3method(roclet_process,roclet_maestroLogLevel) 11 | S3method(roclet_process,roclet_maestroMonths) 12 | S3method(roclet_process,roclet_maestroOutputs) 13 | S3method(roclet_process,roclet_maestroPriority) 14 | S3method(roclet_process,roclet_maestroRunIf) 15 | S3method(roclet_process,roclet_maestroSkip) 16 | S3method(roclet_process,roclet_maestroStartTime) 17 | S3method(roclet_process,roclet_maestroTz) 18 | S3method(roxy_tag_parse,roxy_tag_maestro) 19 | S3method(roxy_tag_parse,roxy_tag_maestroDays) 20 | S3method(roxy_tag_parse,roxy_tag_maestroFlags) 21 | S3method(roxy_tag_parse,roxy_tag_maestroFrequency) 22 | S3method(roxy_tag_parse,roxy_tag_maestroHours) 23 | S3method(roxy_tag_parse,roxy_tag_maestroInputs) 24 | S3method(roxy_tag_parse,roxy_tag_maestroLabel) 25 | S3method(roxy_tag_parse,roxy_tag_maestroLogLevel) 26 | S3method(roxy_tag_parse,roxy_tag_maestroMonths) 27 | S3method(roxy_tag_parse,roxy_tag_maestroOutputs) 28 | S3method(roxy_tag_parse,roxy_tag_maestroPriority) 29 | S3method(roxy_tag_parse,roxy_tag_maestroRunIf) 30 | S3method(roxy_tag_parse,roxy_tag_maestroSkip) 31 | S3method(roxy_tag_parse,roxy_tag_maestroStartTime) 32 | S3method(roxy_tag_parse,roxy_tag_maestroTz) 33 | export(build_schedule) 34 | export(create_maestro) 35 | export(create_pipeline) 36 | export(get_artifacts) 37 | export(get_flags) 38 | export(get_schedule) 39 | export(get_slot_usage) 40 | export(get_status) 41 | export(invoke) 42 | export(last_build_errors) 43 | export(last_run_errors) 44 | export(last_run_messages) 45 | export(last_run_warnings) 46 | export(run_schedule) 47 | export(show_network) 48 | export(suggest_orch_frequency) 49 | import(R6) 50 | import(lifecycle) 51 | import(logger) 52 | import(tictoc) 53 | import(utils) 54 | importFrom(R.utils,countLines) 55 | importFrom(roxygen2,roclet_output) 56 | importFrom(roxygen2,roclet_process) 57 | importFrom(roxygen2,roxy_tag_parse) 58 | -------------------------------------------------------------------------------- /tests/testthat/test-MaestroSchedule.R: -------------------------------------------------------------------------------- 1 | test_that("MaestroSchedule works", { 2 | 3 | schedule <- build_schedule(test_path("test_pipelines_run_all_good")) 4 | 5 | schedule$run( 6 | orch_n = 1, 7 | orch_unit = "day" 8 | ) 9 | 10 | expect_message({ 11 | schedule$run( 12 | orch_n = 1, 13 | orch_unit = "day", 14 | run_all = TRUE 15 | ) 16 | }) 17 | 18 | status <- schedule$get_status() 19 | expect_snapshot(schedule$get_network()) 20 | 21 | expect_s3_class(status, "data.frame") 22 | expect_in( 23 | c("pipe_name", "script_path", "invoked", "success", "pipeline_started", 24 | "pipeline_ended", "errors", "warnings", "messages", "next_run" 25 | ), 26 | names(status) 27 | ) 28 | 29 | artifacts <- schedule$get_artifacts() 30 | 31 | expect_type( 32 | artifacts, "list" 33 | ) 34 | expect_named( 35 | artifacts 36 | ) 37 | expect_gt(length(artifacts), 0) 38 | 39 | expect_s3_class(status$pipeline_started, "POSIXct") 40 | expect_gt(nrow(status), 0) 41 | expect_length(last_run_errors(), 0) 42 | expect_length(last_run_warnings(), 0) 43 | }) |> 44 | suppressMessages() 45 | 46 | test_that("MaestroSchedule correctly returns artifacts (i.e., pipeline returns)", { 47 | 48 | schedule <- build_schedule(test_path("test_pipelines_run_artifacts")) 49 | 50 | output <- schedule$run( 51 | orch_n = 1, 52 | orch_unit = "day", 53 | run_all = TRUE 54 | ) 55 | 56 | artifacts <- schedule$get_artifacts() 57 | 58 | expect_length(artifacts, 1) 59 | expect_equal(artifacts[[1]], "I'm an artifact") 60 | }) |> 61 | suppressMessages() 62 | 63 | test_that("MaestroSchedule works when not running all (verification of checking)", { 64 | 65 | schedule <- build_schedule(test_path("test_pipelines_run_all_good")) 66 | 67 | expect_message({ 68 | schedule$run( 69 | orch_n = 1, 70 | orch_unit = "day" 71 | )} 72 | ) 73 | }) |> 74 | suppressMessages() 75 | 76 | test_that("Multicore works", { 77 | 78 | schedule <- build_schedule(test_path("test_pipelines_run_all_good")) 79 | 80 | expect_no_error({ 81 | schedule$run( 82 | orch_n = 1, 83 | orch_unit = "day", 84 | cores = 2, 85 | run_all = TRUE 86 | ) 87 | }) 88 | }) |> 89 | suppressMessages() 90 | 91 | test_that("MaestroSchedule informs with an empty schedule", { 92 | schedule <- MaestroSchedule$new() 93 | expect_message({ 94 | run_schedule(schedule) 95 | }, "No pipelines") 96 | }) 97 | -------------------------------------------------------------------------------- /tests/testthat/test-create_pipeline.R: -------------------------------------------------------------------------------- 1 | test_that("create_pipeline creates a new pipeline", { 2 | withr::with_tempdir({ 3 | create_pipeline("new_pipe", open = FALSE) 4 | 5 | expect_no_error({ 6 | schedule <- build_schedule() 7 | 8 | run_schedule(schedule) 9 | }) 10 | 11 | expect_true(file.exists("pipelines/new_pipe.R")) 12 | }) |> 13 | expect_message() 14 | }) |> 15 | suppressMessages() 16 | 17 | test_that("create_pipeline fixes bad names", { 18 | withr::with_tempdir({ 19 | create_pipeline("new-pipe", open = FALSE) 20 | 21 | expect_no_error({ 22 | schedule <- build_schedule() 23 | 24 | run_schedule(schedule) 25 | }) 26 | 27 | expect_true(file.exists("pipelines/new_pipe.R")) 28 | }) |> 29 | expect_message() 30 | }) |> 31 | suppressMessages() 32 | 33 | test_that("create_pipeline with POSIXct works and can be run", { 34 | 35 | withr::with_tempdir({ 36 | create_pipeline("new-pipe", start_time = as.POSIXct("2024-05-01 12:00:00"), open = FALSE) 37 | 38 | expect_no_error({ 39 | schedule <- build_schedule() 40 | 41 | run_schedule(schedule) 42 | }) 43 | 44 | expect_true(file.exists("pipelines/new_pipe.R")) 45 | }) |> 46 | expect_message() 47 | 48 | withr::with_tempdir({ 49 | create_pipeline("new-pipe", start_time = "10:00:00", open = FALSE) 50 | 51 | expect_no_error({ 52 | schedule <- build_schedule() 53 | 54 | run_schedule(schedule) 55 | }) 56 | 57 | expect_true(file.exists("pipelines/new_pipe.R")) 58 | }) |> 59 | expect_message() 60 | }) |> 61 | suppressMessages() 62 | 63 | test_that("create_pipeline aborts if pipeline already exists", { 64 | 65 | withr::with_tempdir({ 66 | expect_error({ 67 | create_pipeline("new-pipe", pipeline_dir = ".", open = FALSE) 68 | create_pipeline("new-pipe", pipeline_dir = ".", open = FALSE) 69 | }, regexp = "already exists") 70 | 71 | # Works if overwrite = TRUE 72 | expect_message( 73 | create_pipeline("new-pipe", pipeline_dir = ".", open = FALSE, overwrite = TRUE), 74 | regexp = "Overwriting existing" 75 | ) 76 | }) 77 | }) |> 78 | suppressMessages() 79 | 80 | test_that("create_pipeline with priority", { 81 | withr::with_tempdir({ 82 | create_pipeline("new-pipe", start_time = "10:00:00", open = FALSE, priority = 1, quiet = TRUE) 83 | 84 | expect_no_error({ 85 | schedule <- build_schedule(quiet = TRUE) 86 | 87 | run_schedule(schedule, quiet = TRUE) 88 | }) 89 | 90 | expect_true(file.exists("pipelines/new_pipe.R")) 91 | }) 92 | }) 93 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/MaestroPipeline.md: -------------------------------------------------------------------------------- 1 | # Simple pipeline, no errors 2 | 3 | Code 4 | pipeline$get_status() 5 | Output 6 | # A tibble: 1 x 9 7 | pipe_name script_path invoked success pipeline_started pipeline_ended 8 | 9 | 1 get_mtcars test_pipel~ FALSE FALSE NA NA 10 | # i 3 more variables: errors , warnings , messages 11 | 12 | --- 13 | 14 | Code 15 | pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")] 16 | Output 17 | # A tibble: 1 x 5 18 | invoked success errors warnings messages 19 | 20 | 1 TRUE TRUE 0 0 0 21 | 22 | # Pipeline with warnings 23 | 24 | Code 25 | pipeline$get_status() 26 | Output 27 | # A tibble: 1 x 9 28 | pipe_name script_path invoked success pipeline_started pipeline_ended 29 | 30 | 1 pipe3 test_pipeli~ FALSE FALSE NA NA 31 | # i 3 more variables: errors , warnings , messages 32 | 33 | --- 34 | 35 | Code 36 | pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")] 37 | Output 38 | # A tibble: 1 x 5 39 | invoked success errors warnings messages 40 | 41 | 1 TRUE TRUE 0 1 0 42 | 43 | # Pipeline with errors 44 | 45 | Code 46 | pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")] 47 | Output 48 | # A tibble: 1 x 5 49 | invoked success errors warnings messages 50 | 51 | 1 TRUE FALSE 1 0 0 52 | 53 | # Pipeline with arguments are correctly passed 54 | 55 | Code 56 | pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")] 57 | Output 58 | # A tibble: 1 x 5 59 | invoked success errors warnings messages 60 | 61 | 1 FALSE FALSE 0 0 0 62 | 63 | --- 64 | 65 | Code 66 | pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")] 67 | Output 68 | # A tibble: 1 x 5 69 | invoked success errors warnings messages 70 | 71 | 1 TRUE TRUE 0 0 0 72 | 73 | -------------------------------------------------------------------------------- /man/create_pipeline.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/create_pipeline.R 3 | \name{create_pipeline} 4 | \alias{create_pipeline} 5 | \title{Create a new pipeline in a pipelines directory} 6 | \usage{ 7 | create_pipeline( 8 | pipe_name, 9 | pipeline_dir = "pipelines", 10 | frequency = "1 day", 11 | start_time = Sys.Date(), 12 | tz = "UTC", 13 | log_level = "INFO", 14 | quiet = FALSE, 15 | open = interactive(), 16 | overwrite = FALSE, 17 | skip = FALSE, 18 | inputs = NULL, 19 | outputs = NULL, 20 | priority = NULL 21 | ) 22 | } 23 | \arguments{ 24 | \item{pipe_name}{name of the pipeline and function} 25 | 26 | \item{pipeline_dir}{directory containing the pipeline scripts} 27 | 28 | \item{frequency}{how often the pipeline should run (e.g., 1 day, daily, 3 hours, 4 months). Fills in maestroFrequency tag} 29 | 30 | \item{start_time}{start time of the pipeline schedule. Fills in maestroStartTime tag} 31 | 32 | \item{tz}{timezone that pipeline will be scheduled in. Fills in maestroTz tag} 33 | 34 | \item{log_level}{log level for the pipeline (e.g., INFO, WARN, ERROR). Fills in maestroLogLevel tag} 35 | 36 | \item{quiet}{whether to silence messages in the console (default = \code{FALSE})} 37 | 38 | \item{open}{whether or not to open the script upon creation} 39 | 40 | \item{overwrite}{whether or not to overwrite an existing pipeline of the same name and location.} 41 | 42 | \item{skip}{whether to skip the pipeline when running in the orchestrator (default = \code{FALSE})} 43 | 44 | \item{inputs}{vector of names of pipelines that input into this pipeline (default = \code{NULL} for no inputs)} 45 | 46 | \item{outputs}{vector of names of pipelines that receive output from this pipeline (default = \code{NULL} for no outputs)} 47 | 48 | \item{priority}{a single positive integer corresponding to the order in which this pipeline will be invoked in the presence of other simultaneously invoked pipelines.} 49 | } 50 | \value{ 51 | invisible 52 | } 53 | \description{ 54 | Allows the creation of new pipelines (R scripts) and fills in the maestro tags as specified. 55 | } 56 | \examples{ 57 | if (interactive()) { 58 | pipeline_dir <- tempdir() 59 | create_pipeline( 60 | "extract_data", 61 | pipeline_dir = pipeline_dir, 62 | frequency = "1 hour", 63 | open = FALSE, 64 | quiet = TRUE, 65 | overwrite = TRUE 66 | ) 67 | 68 | create_pipeline( 69 | "new_job", 70 | pipeline_dir = pipeline_dir, 71 | frequency = "20 minutes", 72 | start_time = as.POSIXct("2024-06-21 12:20:00"), 73 | log_level = "ERROR", 74 | open = FALSE, 75 | quiet = TRUE, 76 | overwrite = TRUE 77 | ) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /tests/testthat/test-MaestroPipeline.R: -------------------------------------------------------------------------------- 1 | test_that("Simple pipeline, no errors", { 2 | 3 | pipeline_list <- build_schedule_entry( 4 | test_path("test_pipelines/test_pipeline_daily_good.R") 5 | ) 6 | 7 | pipeline <- pipeline_list$MaestroPipelines[[1]] 8 | expect_s3_class(pipeline, "MaestroPipeline") 9 | 10 | expect_snapshot(pipeline$get_status()) 11 | expect_null(pipeline$get_outputs()) 12 | expect_null(pipeline$get_inputs()) 13 | 14 | pipeline$run(quiet = TRUE) 15 | 16 | expect_snapshot(pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")]) 17 | }) 18 | 19 | test_that("Pipeline with warnings", { 20 | 21 | pipeline_list <- build_schedule_entry( 22 | test_path("test_pipelines_run_two_warnings/pipe_warning.R") 23 | ) 24 | pipeline <- pipeline_list$MaestroPipelines[[1]] 25 | 26 | expect_snapshot(pipeline$get_status()) 27 | 28 | pipeline$run(quiet = TRUE) 29 | 30 | expect_snapshot(pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")]) 31 | }) 32 | 33 | test_that("Pipeline with errors", { 34 | 35 | pipeline_list <- build_schedule_entry( 36 | test_path("test_pipelines_run_some_errors/pipe2.R") 37 | ) 38 | pipeline <- pipeline_list$MaestroPipelines[[1]] 39 | 40 | expect_error(pipeline$run(quiet = TRUE)) 41 | 42 | expect_snapshot(pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")]) 43 | }) 44 | 45 | test_that("Pipeline with arguments are correctly passed", { 46 | 47 | pipeline_list <- build_schedule_entry( 48 | test_path("test_pipelines_run_args_good/pipe1.R") 49 | ) 50 | 51 | pipeline <- pipeline_list$MaestroPipelines[[1]] 52 | 53 | expect_snapshot(pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")]) 54 | 55 | pipeline$run(resources = list( 56 | vals = 1:5 57 | ), quiet = TRUE) 58 | 59 | expect_snapshot(pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")]) 60 | }) 61 | 62 | test_that("Logging threshold", { 63 | 64 | pipeline_list <- build_schedule_entry( 65 | test_path("test_pipelines_run_logs_warn/pipe_warning.R") 66 | ) 67 | 68 | pipeline <- pipeline_list$MaestroPipelines[[1]] 69 | 70 | withr::with_tempfile("log", { 71 | 72 | pipeline$run( 73 | log_file = log, 74 | quiet = TRUE 75 | ) 76 | 77 | logs <- readLines(log) 78 | expect_true(!all(grepl("INFO", logs))) 79 | expect_true(any(grepl("WARN", logs))) 80 | }) 81 | 82 | withr::with_tempfile("log", { 83 | pipeline$run( 84 | log_file = log, 85 | log_file_max_bytes = 1000, 86 | quiet = TRUE 87 | ) 88 | 89 | expect_lte(file.size(log), 1000 + 100) # margin of error 90 | }) 91 | }) 92 | 93 | 94 | -------------------------------------------------------------------------------- /tests/testthat/test-get_slot_usage.R: -------------------------------------------------------------------------------- 1 | test_that("get_slot_usage works as expected", { 2 | withr::with_tempdir({ 3 | dir.create("pipelines") 4 | writeLines( 5 | " 6 | #' @maestroFrequency hourly 7 | #' @maestroStartTime 10:00:00 8 | hourly1 <- function() { 9 | } 10 | 11 | #' @maestroFrequency 2 hours 12 | #' @maestroStartTime 10:00:00 13 | hourly2 <- function() { 14 | } 15 | 16 | #' @maestroFrequency 3 hours 17 | #' @maestroStartTime 10:00:00 18 | #' @maestroTz America/Halifax 19 | hourly3 <- function() { 20 | } 21 | ", 22 | con = "pipelines/hourlies.R" 23 | ) 24 | 25 | schedule <- build_schedule(quiet = TRUE) 26 | }) 27 | 28 | avail_hour <- get_slot_usage( 29 | schedule, 30 | orch_frequency = "1 hour" 31 | ) 32 | expect_snapshot(avail_hour) 33 | 34 | avail_day <- get_slot_usage( 35 | schedule, 36 | orch_frequency = "1 hour", 37 | slot_interval = "day" 38 | ) 39 | 40 | expect_snapshot(avail_day) 41 | }) 42 | 43 | test_that("get_slot_usage works with variety of frequencies", { 44 | 45 | withr::with_tempdir({ 46 | dir.create("pipelines") 47 | writeLines( 48 | " 49 | #' @maestroFrequency hourly 50 | #' @maestroStartTime 10:00:00 51 | hourly1 <- function() { 52 | } 53 | 54 | #' @maestroFrequency 3 days 55 | #' @maestroStartTime 2025-01-01 12:00:00 56 | daily1 <- function() { 57 | } 58 | 59 | #' @maestroFrequency weekly 60 | #' @maestroStartTime 2025-01-02 10:00:00 61 | #' @maestroTz UTC 62 | weekly1 <- function() { 63 | } 64 | ", 65 | con = "pipelines/multi.R" 66 | ) 67 | 68 | schedule <- build_schedule(quiet = TRUE) 69 | }) 70 | 71 | avail_hour <- get_slot_usage( 72 | schedule, 73 | orch_frequency = "1 hour" 74 | ) 75 | expect_snapshot(avail_hour) 76 | 77 | avail_day <- get_slot_usage( 78 | schedule, 79 | orch_frequency = "1 hour", 80 | slot_interval = "day" 81 | ) 82 | 83 | expect_snapshot(avail_day) 84 | 85 | avail_week <- get_slot_usage( 86 | schedule, 87 | orch_frequency = "1 hour", 88 | slot_interval = "week" 89 | ) 90 | 91 | expect_snapshot(avail_week) 92 | }) 93 | 94 | test_that("get_slot_usage informs on empty schedule", { 95 | withr::with_tempdir({ 96 | dir.create("pipelines") 97 | writeLines( 98 | "", 99 | con = "pipelines/multi.R" 100 | ) 101 | 102 | schedule <- build_schedule(quiet = TRUE) 103 | }) 104 | 105 | expect_message({ 106 | get_slot_usage( 107 | schedule, 108 | "1 hour" 109 | ) 110 | }) 111 | }) 112 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/run_schedule.md: -------------------------------------------------------------------------------- 1 | # run_schedule timeliness checks - pipelines run when they're supposed to 2 | 3 | Code 4 | status$invoked 5 | Output 6 | [1] FALSE FALSE FALSE FALSE FALSE FALSE TRUE 7 | 8 | --- 9 | 10 | Code 11 | status$next_run 12 | Output 13 | [1] "2024-04-26 09:00:00 UTC" "2024-04-25 12:00:00 UTC" 14 | [3] "2024-07-01 00:00:00 UTC" "2024-04-29 00:00:00 UTC" 15 | [5] "2030-12-12 10:15:00 UTC" "2024-04-29 00:00:00 UTC" 16 | [7] "2024-04-25 10:30:00 UTC" 17 | 18 | --- 19 | 20 | Code 21 | status$invoked 22 | Output 23 | [1] TRUE FALSE TRUE TRUE FALSE TRUE TRUE 24 | 25 | --- 26 | 27 | Code 28 | status$next_run 29 | Output 30 | [1] "2024-05-01 00:00:00 UTC" "2024-04-01 03:00:00 UTC" 31 | [3] "2024-07-01 00:00:00 UTC" "2024-05-01 00:00:00 UTC" 32 | [5] "2030-12-01 00:00:00 UTC" "2024-05-01 00:00:00 UTC" 33 | [7] "2024-05-01 00:00:00 UTC" 34 | 35 | --- 36 | 37 | Code 38 | status$invoked 39 | Output 40 | [1] TRUE FALSE TRUE TRUE FALSE TRUE TRUE 41 | 42 | --- 43 | 44 | Code 45 | status$next_run 46 | Output 47 | [1] "2024-04-05 00:00:00 UTC" "2024-04-01 03:00:00 UTC" 48 | [3] "2024-07-01 00:00:00 UTC" "2024-04-09 00:00:00 UTC" 49 | [5] "2030-12-13 00:00:00 UTC" "2024-04-09 00:00:00 UTC" 50 | [7] "2024-04-05 00:00:00 UTC" 51 | 52 | --- 53 | 54 | Code 55 | status$invoked 56 | Output 57 | [1] FALSE TRUE FALSE FALSE FALSE FALSE TRUE 58 | 59 | --- 60 | 61 | Code 62 | status$next_run 63 | Output 64 | [1] "2024-03-03 09:00:00 UTC" "2024-03-03 13:00:00 UTC" 65 | [3] "2024-04-01 00:00:00 UTC" "2024-03-04 00:00:00 UTC" 66 | [5] "2030-12-12 10:00:00 UTC" "2024-03-04 00:00:00 UTC" 67 | [7] "2024-03-02 14:00:00 UTC" 68 | 69 | # run_schedule timeliness checks - specifiers (e.g., hours, days, months) 70 | 71 | Code 72 | status$invoked 73 | Output 74 | [1] TRUE TRUE FALSE TRUE FALSE FALSE FALSE 75 | 76 | --- 77 | 78 | Code 79 | status$next_run 80 | Output 81 | [1] "2024-04-08 00:00:00 UTC" "2024-04-01 04:00:00 UTC" 82 | [3] "2024-05-06 00:00:00 UTC" "2024-04-01 01:00:00 UTC" 83 | [5] "2024-04-06 00:00:00 UTC" "2024-05-01 00:00:00 UTC" 84 | [7] "2024-05-01 01:00:00 UTC" 85 | 86 | --- 87 | 88 | Code 89 | status$invoked 90 | Output 91 | [1] TRUE TRUE FALSE TRUE FALSE TRUE FALSE 92 | 93 | --- 94 | 95 | Code 96 | status$next_run 97 | Output 98 | [1] "2024-05-08 00:00:00 UTC" "2024-05-01 04:00:00 UTC" 99 | [3] "2024-05-06 00:00:00 UTC" "2024-05-01 01:00:00 UTC" 100 | [5] "2024-05-04 00:00:00 UTC" "2024-10-01 00:00:00 UTC" 101 | [7] "2024-05-01 01:00:00 UTC" 102 | 103 | # maestroStartTime with HH:MM:SS runs on the expected time 104 | 105 | Code 106 | status$invoked 107 | Output 108 | [1] TRUE 109 | 110 | # maestroPriority works as expected 111 | 112 | Code 113 | status$pipe_name 114 | Output 115 | [1] "q1" "p2" 116 | 117 | -------------------------------------------------------------------------------- /R/suggest_orch_frequency.R: -------------------------------------------------------------------------------- 1 | #' Suggest orchestrator frequency based on a schedule 2 | #' 3 | #' Suggests a frequency to run the orchestrator based on the frequencies of the 4 | #' pipelines in a schedule. 5 | #' 6 | #' This function attempts to find the smallest interval of time between all pipelines. 7 | #' If the smallest interval is less than 15 minutes, it just uses the smallest interval. 8 | #' 9 | #' Note this function is intended to be used interactively when deciding how often to 10 | #' schedule the orchestrator. Programmatic use is not recommended. 11 | #' 12 | #' @param schedule MaestroSchedule object created by `build_schedule()` 13 | #' @inheritParams get_pipeline_run_sequence 14 | #' 15 | #' @return frequency string 16 | #' @export 17 | #' 18 | #' @examples 19 | #' 20 | #' if (interactive()) { 21 | #' pipeline_dir <- tempdir() 22 | #' create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 23 | #' schedule <- build_schedule(pipeline_dir = pipeline_dir) 24 | #' suggest_orch_frequency(schedule) 25 | #' } 26 | suggest_orch_frequency <- function(schedule, check_datetime = lubridate::now(tzone = "UTC")) { 27 | 28 | # Check that schedule is a MaestroSchedule 29 | if (!"MaestroSchedule" %in% class(schedule)) { 30 | cli::cli_abort( 31 | c("Schedule must be an object of {.cls MaestroSchedule} and not an object of class {.cls {class(schedule)}}.", 32 | "i" = "Use {.fn build_schedule} to create a valid schedule."), 33 | call = rlang::caller_env() 34 | ) 35 | } 36 | 37 | schedule <- schedule$get_schedule() |> 38 | dplyr::filter(!skip, !is.na(frequency_n)) 39 | 40 | if (nrow(schedule) == 0) { 41 | 42 | cli::cli_abort( 43 | c( 44 | "No pipelines in schedule after removing skipped pipelines.", 45 | "i" = "Remove `maestroSkip` tags to get a suggested frequency." 46 | ) 47 | ) 48 | } 49 | 50 | sch_secs <- purrr::map_int( 51 | paste(schedule$frequency_n, schedule$frequency_unit), 52 | purrr::possibly(convert_to_seconds, otherwise = NA, quiet = TRUE) 53 | ) 54 | 55 | # If the minimum schedule seconds is lte 15 minutes, return the corresponding frequency 56 | if (min(sch_secs, na.rm = TRUE) <= (60 * 15)) { 57 | return(schedule$frequency[[which.min(sch_secs)]]) 58 | } 59 | 60 | if (nrow(schedule) == 1) { 61 | return(schedule$frequency) 62 | } 63 | 64 | max_idx <- which.max(sch_secs) 65 | max_freq <- paste(schedule$frequency_n[[max_idx]], schedule$frequency_unit[[max_idx]]) 66 | 67 | pipeline_sequences <- purrr::pmap( 68 | list(schedule$frequency_n, schedule$frequency_unit, schedule$start_time), 69 | ~{ 70 | pipeline_sequence <- get_pipeline_run_sequence( 71 | ..1, ..2, ..3, 72 | check_datetime = check_datetime + lubridate::seconds(convert_to_seconds(max_freq)) 73 | ) 74 | 75 | pipeline_sequence[pipeline_sequence >= check_datetime] 76 | } 77 | ) |> 78 | purrr::list_c() |> 79 | unique() |> 80 | sort() 81 | 82 | if (length(pipeline_sequences) == 1) return(max_freq) 83 | 84 | pipeline_diffs <- diff(pipeline_sequences) 85 | 86 | min_diff_secs <- min(pipeline_diffs) 87 | 88 | min_diff_atts <- attributes(min_diff_secs) 89 | 90 | paste(round(as.numeric(min_diff_secs)), min_diff_atts$units) 91 | } 92 | -------------------------------------------------------------------------------- /tests/testthat/test-MaestroPipelineList.R: -------------------------------------------------------------------------------- 1 | test_that("Simple pipeline list, no errors", { 2 | 3 | pipeline_list <- build_schedule_entry( 4 | test_path("test_pipelines/test_pipeline_daily_good.R") 5 | ) 6 | 7 | expect_s3_class(pipeline_list, "MaestroPipelineList") 8 | expect_snapshot(pipeline_list) 9 | expect_type(pipeline_list$get_pipe_names(), "character") 10 | expect_s3_class(pipeline_list$get_pipe_by_name("get_mtcars"), "MaestroPipeline") 11 | expect_error( 12 | pipeline_list$get_pipe_by_name("asdasd"), 13 | regexp = "No pipeline" 14 | ) 15 | 16 | pipeline_list$run(quiet = TRUE) 17 | pipeline <- pipeline_list$MaestroPipelines[[1]] 18 | expect_snapshot(pipeline$get_status()[c("invoked", "success", "errors", "warnings", "messages")]) 19 | 20 | expect_type( 21 | pipeline_list$check_timeliness(orch_n = 1, orch_unit = "day"), 22 | "logical" 23 | ) 24 | 25 | expect_s3_class( 26 | pipeline_list$get_timely_pipelines(orch_n = 1, orch_unit = "day"), 27 | "MaestroPipelineList" 28 | ) 29 | }) 30 | 31 | test_that("Errors are handled", { 32 | 33 | pipeline_list <- build_schedule_entry( 34 | test_path("test_pipelines_run_some_errors/pipe2.R") 35 | ) 36 | 37 | expect_no_error(pipeline_list$run(quiet = TRUE)) 38 | }) 39 | 40 | test_that("Populate on instantiation", { 41 | 42 | pipeline_list <- build_schedule_entry( 43 | test_path("test_pipelines/test_pipeline_daily_good.R") 44 | ) 45 | 46 | pipelines <- pipeline_list$MaestroPipelines 47 | 48 | new_pipeline_list <- MaestroPipelineList$new(pipelines) 49 | expect_s3_class(new_pipeline_list, "MaestroPipelineList") 50 | expect_s3_class(new_pipeline_list$MaestroPipelines[[1]], "MaestroPipeline") 51 | }) 52 | 53 | test_that("Populate after instantiation", { 54 | pipeline_list <- build_schedule_entry( 55 | test_path("test_pipelines/test_pipeline_daily_good.R") 56 | ) 57 | 58 | pipelines <- pipeline_list$MaestroPipelines 59 | 60 | new_pipeline_list <- MaestroPipelineList$new() 61 | new_pipeline_list$add_pipelines(pipelines[[1]]) 62 | expect_s3_class(new_pipeline_list, "MaestroPipelineList") 63 | expect_s3_class(new_pipeline_list$MaestroPipelines[[1]], "MaestroPipeline") 64 | expect_equal(new_pipeline_list$n_pipelines, 1) 65 | }) 66 | 67 | test_that("Attribute n_pipelines is valid", { 68 | pipeline_list1 <- build_schedule_entry( 69 | test_path("test_pipelines/test_pipeline_daily_good.R") 70 | ) 71 | 72 | expect_equal(pipeline_list1$n_pipelines, 1) 73 | 74 | pipeline_list2 <- build_schedule_entry( 75 | test_path("test_pipelines/test_multi_fun_pipeline.R") 76 | ) 77 | 78 | expect_equal(pipeline_list2$n_pipelines, 2) 79 | 80 | new_pipe <- build_schedule_entry( 81 | test_path("test_pipelines/test_pipeline_hours_good.R") 82 | ) 83 | 84 | pipeline_list2$add_pipelines(new_pipe) 85 | 86 | expect_equal(pipeline_list2$n_pipelines, 3) 87 | }) 88 | 89 | test_that("Can add two MaestroPipelineLists", { 90 | pipeline_list1 <- build_schedule_entry( 91 | test_path("test_pipelines/test_pipeline_daily_good.R") 92 | ) 93 | 94 | pipeline_list2 <- build_schedule_entry( 95 | test_path("test_pipelines/test_multi_fun_pipeline.R") 96 | ) 97 | 98 | pipeline_list1$add_pipelines(pipeline_list2) 99 | expect_equal(pipeline_list1$n_pipelines, 3) 100 | }) 101 | -------------------------------------------------------------------------------- /R/build_schedule.R: -------------------------------------------------------------------------------- 1 | #' Build a schedule table 2 | #' 3 | #' Builds a schedule data.frame for scheduling pipelines in `run_schedule()`. 4 | #' 5 | #' This function parses the maestro tags of functions located in `pipeline_dir` which is 6 | #' conventionally called 'pipelines'. An orchestrator requires a schedule table 7 | #' to determine which pipelines are to run and when. Each row in a schedule table 8 | #' is a pipeline name and its scheduling parameters such as its frequency. 9 | #' 10 | #' The schedule table is mostly intended to be used by `run_schedule()` immediately. 11 | #' In other words, it is not recommended to make changes to it. 12 | #' 13 | #' @param pipeline_dir path to directory containing the pipeline scripts 14 | #' @param quiet silence metrics to the console (default = `FALSE`) 15 | #' 16 | #' @return MaestroSchedule 17 | #' @export 18 | #' @examples 19 | #' 20 | #' # Creating a temporary directory for demo purposes! In practice, just 21 | #' # create a 'pipelines' directory at the project level. 22 | #' 23 | #' if (interactive()) { 24 | #' pipeline_dir <- tempdir() 25 | #' create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 26 | #' build_schedule(pipeline_dir = pipeline_dir) 27 | #' } 28 | build_schedule <- function(pipeline_dir = "./pipelines", quiet = FALSE) { 29 | 30 | if (!dir.exists(pipeline_dir)) { 31 | cli::cli_abort("No directory called {.emph {pipeline_dir}}") 32 | } 33 | 34 | # Parse all the .R files in the `pipeline_dir` directory 35 | pipelines <- list.files( 36 | pipeline_dir, 37 | pattern = "*.R$", 38 | full.names = TRUE 39 | ) 40 | 41 | # Error if the directory has no .R scripts 42 | if(length(pipelines) == 0) { 43 | cli::cli_inform( 44 | "No R scripts in {pipeline_dir}." 45 | ) 46 | return(invisible()) 47 | } 48 | 49 | # Try to generate a schedule entry for each script 50 | # We use safely to ensure it continues in an error condition and capture the errors 51 | pipeline_attempts <- purrr::map( 52 | pipelines, purrr::safely(build_schedule_entry) 53 | ) |> 54 | stats::setNames(basename(pipelines)) 55 | 56 | # Get the results 57 | pipeline_results <- purrr::map( 58 | pipeline_attempts, 59 | ~.x$result 60 | ) |> 61 | purrr::discard(is.null) 62 | 63 | # Get the errors 64 | pipeline_errors <- purrr::map( 65 | pipeline_attempts, 66 | ~.x$error 67 | ) |> 68 | purrr::discard(is.null) 69 | 70 | # Assign the errors to the pkgenv 71 | maestro_pkgenv$last_build_errors <- pipeline_errors 72 | 73 | # Check uniqueness of the function names 74 | pipe_names <- purrr::map(pipeline_results, ~{ 75 | .x$get_pipe_names() 76 | }) |> 77 | purrr::list_c() 78 | 79 | # Check for uniqueness of pipe names 80 | if (length(unique(pipe_names)) < length(pipe_names)) { 81 | non_unique_names <- pipe_names[duplicated(pipe_names)] 82 | cli::cli_abort( 83 | c("Function names must all be unique", 84 | "i" = "{.code {non_unique_names}} used more than once.") 85 | ) 86 | } 87 | 88 | # Create the schedule 89 | schedule <- MaestroSchedule$new(Pipelines = pipeline_results) 90 | 91 | # Validate the schedule 92 | schedule$PipelineList$validate_network() 93 | 94 | if (!quiet) { 95 | maestro_parse_cli(pipeline_results, pipeline_errors) 96 | } 97 | 98 | schedule 99 | } 100 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/get_slot_usage.md: -------------------------------------------------------------------------------- 1 | # get_slot_usage works as expected 2 | 3 | Code 4 | avail_hour 5 | Output 6 | # A tibble: 24 x 3 7 | slot n_runs pipe_names 8 | 9 | 1 00:00 2 hourly1, hourly2 10 | 2 01:00 2 hourly1, hourly3 11 | 3 02:00 3 hourly1, hourly2, hourly3 12 | 4 03:00 1 hourly1 13 | 5 04:00 3 hourly1, hourly2, hourly3 14 | 6 05:00 2 hourly1, hourly3 15 | 7 06:00 2 hourly1, hourly2 16 | 8 07:00 2 hourly1, hourly3 17 | 9 08:00 3 hourly1, hourly2, hourly3 18 | 10 09:00 1 hourly1 19 | # i 14 more rows 20 | 21 | --- 22 | 23 | Code 24 | avail_day 25 | Output 26 | # A tibble: 31 x 3 27 | slot n_runs pipe_names 28 | 29 | 1 01 3 hourly1, hourly2, hourly3 30 | 2 02 3 hourly1, hourly2, hourly3 31 | 3 03 3 hourly1, hourly2, hourly3 32 | 4 04 3 hourly1, hourly2, hourly3 33 | 5 05 3 hourly1, hourly2, hourly3 34 | 6 06 3 hourly1, hourly2, hourly3 35 | 7 07 3 hourly1, hourly2, hourly3 36 | 8 08 3 hourly1, hourly2, hourly3 37 | 9 09 3 hourly1, hourly2, hourly3 38 | 10 10 3 hourly1, hourly2, hourly3 39 | # i 21 more rows 40 | 41 | # get_slot_usage works with variety of frequencies 42 | 43 | Code 44 | avail_hour 45 | Output 46 | # A tibble: 24 x 3 47 | slot n_runs pipe_names 48 | 49 | 1 00:00 1 hourly1 50 | 2 01:00 1 hourly1 51 | 3 02:00 1 hourly1 52 | 4 03:00 1 hourly1 53 | 5 04:00 1 hourly1 54 | 6 05:00 1 hourly1 55 | 7 06:00 1 hourly1 56 | 8 07:00 1 hourly1 57 | 9 08:00 1 hourly1 58 | 10 09:00 1 hourly1 59 | # i 14 more rows 60 | 61 | --- 62 | 63 | Code 64 | avail_day 65 | Output 66 | # A tibble: 31 x 3 67 | slot n_runs pipe_names 68 | 69 | 1 01 3 hourly1, daily1, weekly1 70 | 2 02 3 hourly1, daily1, weekly1 71 | 3 03 3 hourly1, daily1, weekly1 72 | 4 04 3 hourly1, daily1, weekly1 73 | 5 05 3 hourly1, daily1, weekly1 74 | 6 06 3 hourly1, daily1, weekly1 75 | 7 07 3 hourly1, daily1, weekly1 76 | 8 08 3 hourly1, daily1, weekly1 77 | 9 09 3 hourly1, daily1, weekly1 78 | 10 10 3 hourly1, daily1, weekly1 79 | # i 21 more rows 80 | 81 | --- 82 | 83 | Code 84 | avail_week 85 | Output 86 | # A tibble: 7 x 3 87 | slot n_runs pipe_names 88 | 89 | 1 Fri 2 hourly1, daily1 90 | 2 Mon 2 hourly1, daily1 91 | 3 Sat 2 hourly1, daily1 92 | 4 Sun 2 hourly1, daily1 93 | 5 Thu 3 hourly1, daily1, weekly1 94 | 6 Tue 2 hourly1, daily1 95 | 7 Wed 2 hourly1, daily1 96 | 97 | -------------------------------------------------------------------------------- /R/get_slot_usage.R: -------------------------------------------------------------------------------- 1 | #' Get time slot usage of a schedule 2 | #' 3 | #' Get the number of pipelines scheduled to run for each time slot at a particular 4 | #' slot interval. Time slots are times that the orchestrator runs and the slot interval 5 | #' determines the level of granularity to consider. 6 | #' 7 | #' This function is particularly useful when you have multiple pipelines in a project 8 | #' and you want to see what recurring time intervals may be available or underused 9 | #' for new pipelines. 10 | #' 11 | #' Note that this function is intended for use in an interactive session while developing 12 | #' a maestro project. It is not intended for use in the orchestrator. 13 | #' 14 | #' As an example, consider we have four pipelines running at various frequencies 15 | #' and the orchestrator running every hour. Then let's say there's to be a new 16 | #' pipeline that runs every day. One might ask 'what hour should I schedule this new 17 | #' pipeline to run on?'. By using `get_slot_usage(schedule, orch_frequency = '1 hour', slot_interval = 'hour')` 18 | #' on the existing schedule, you could identify for each hour how many pipelines 19 | #' are already scheduled to run and choose the ones with the lowest usage. 20 | #' 21 | #' @inheritParams run_schedule 22 | #' @param slot_interval a time unit indicating the interval of time to consider between slots (e.g., 'hour', 'day') 23 | #' 24 | #' @returns data.frame 25 | #' @export 26 | #' 27 | #' @examples 28 | #' if (interactive()) { 29 | #' pipeline_dir <- tempdir() 30 | #' create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 31 | #' schedule <- build_schedule(pipeline_dir = pipeline_dir) 32 | #' 33 | #' get_slot_usage( 34 | #' schedule, 35 | #' orch_frequency = "1 hour", 36 | #' slot_interval = "hour" 37 | #' ) 38 | #' } 39 | get_slot_usage <- function(schedule, orch_frequency, slot_interval = "hour") { 40 | 41 | if (!"MaestroSchedule" %in% class(schedule)) { 42 | cli::cli_abort( 43 | c("Schedule must be an object of {.cls MaestroSchedule} and not an object of class {.cls {class(schedule)}}.", 44 | "i" = "Use {.fn build_schedule} to create a valid schedule."), 45 | call = rlang::caller_env() 46 | ) 47 | } 48 | 49 | if (length(schedule$PipelineList$MaestroPipelines) == 0) { 50 | cli::cli_inform("No pipelines in schedule.") 51 | return(invisible()) 52 | } 53 | 54 | # Get the orchestrator nunits 55 | orch_nunits <- validate_orch_frequency(orch_frequency) 56 | 57 | window_std <- standardize_units(slot_interval) 58 | 59 | if (!window_std %in% valid_units) { 60 | cli::cli_abort( 61 | "`slot_interval` must be a valid time unit such as 'hour', 'day', 'week', etc." 62 | ) 63 | } 64 | 65 | run_sequences <- schedule$PipelineList$get_run_sequences() 66 | 67 | run_sequences_all <- run_sequences |> 68 | purrr::imap( 69 | ~dplyr::tibble( 70 | pipe_name = .y, 71 | slot = .x 72 | ) 73 | ) |> 74 | purrr::list_rbind() 75 | 76 | window_fmt <- switch (window_std, 77 | second = "%S", 78 | minute = "%M", 79 | hour = "%H:%M", 80 | day = "%d", 81 | week = "%a", 82 | month = "%b", 83 | quarter = "%b", 84 | year = "%Y" 85 | ) 86 | 87 | run_sequences_all |> 88 | dplyr::filter(!is.na(slot)) |> 89 | dplyr::mutate( 90 | slot = format(slot, window_fmt) 91 | ) |> 92 | dplyr::distinct(pipe_name, slot) |> 93 | dplyr::summarise( 94 | n_runs = dplyr::n(), 95 | pipe_names = paste(pipe_name, collapse = ", "), 96 | .by = slot 97 | ) |> 98 | dplyr::arrange(slot) 99 | } 100 | -------------------------------------------------------------------------------- /tests/testthat/test-invoke.R: -------------------------------------------------------------------------------- 1 | schedule <- build_schedule(test_path("test_pipelines_run_all_good"), quiet = TRUE) 2 | 3 | test_that("invoke errors if schedule is not a MaestroSchedule", { 4 | expect_error({ 5 | invoke(iris, pipe_name = "hello") 6 | }, regexp = "Schedule must be an object") 7 | }) 8 | 9 | test_that("invoke errors if pipe_name is not a single character", { 10 | 11 | expect_error({ 12 | invoke(schedule, pipe_name = letters) 13 | }, regexp = "must be a single character") 14 | }) 15 | 16 | test_that("invoke errors if pipe_name is not in the schedule", { 17 | expect_error({ 18 | invoke(schedule, pipe_name = "I don't exist") 19 | }, regexp = "is not the name of a pipeline in the schedule") 20 | }) 21 | 22 | test_that("errors if resources are unnamed or non unique", { 23 | 24 | expect_error({ 25 | invoke(schedule, "chatty", quiet = TRUE, resources = list(4)) 26 | }, regexp = "All elements") 27 | 28 | expect_error({ 29 | invoke(schedule, "chatty", quiet = TRUE, resources = list(a = 1, a = 2)) 30 | }, regexp = "All elements") 31 | }) 32 | 33 | test_that("invoke triggers the pipeline", { 34 | 35 | expect_no_error({ 36 | invoke(schedule, "chatty", quiet = TRUE) 37 | }) 38 | 39 | status <- schedule$get_status() 40 | expect_true(status$invoked[status$pipe_name == "chatty"]) 41 | }) 42 | 43 | test_that("invoke properly passes resources", { 44 | 45 | withr::with_tempdir({ 46 | dir.create("pipelines") 47 | writeLines( 48 | " 49 | #' @maestroFrequency hourly 50 | times2 <- function(val) { 51 | val * 2 52 | } 53 | ", 54 | con = "pipelines/invoked.R" 55 | ) 56 | 57 | schedule <- build_schedule(quiet = TRUE) 58 | 59 | invoke(schedule, "times2", resources = list(val = 2), quiet = TRUE) 60 | 61 | expect_true(get_status(schedule)$success) 62 | }) 63 | }) 64 | 65 | test_that("invoke gives informative error message on failure", { 66 | 67 | withr::with_tempdir({ 68 | dir.create("pipelines") 69 | writeLines( 70 | " 71 | #' @maestroFrequency hourly 72 | i_fail <- function() { 73 | stop() 74 | } 75 | ", 76 | con = "pipelines/invoked.R" 77 | ) 78 | 79 | schedule <- build_schedule(quiet = TRUE) 80 | 81 | invoke(schedule, "i_fail", quiet = TRUE) 82 | 83 | expect_snapshot(schedule$get_status()[, c("invoked", "success")]) 84 | }) 85 | }) 86 | 87 | test_that("invoke triggers DAG pipelines", { 88 | 89 | withr::with_tempdir({ 90 | dir.create("pipelines") 91 | writeLines( 92 | " 93 | #' @maestroFrequency hourly 94 | p1 <- function() { 95 | 2 96 | } 97 | 98 | #' @maestroInputs p1 99 | p2 <- function(.input) { 100 | .input * 2 101 | } 102 | ", 103 | con = "pipelines/dag.R" 104 | ) 105 | 106 | schedule <- build_schedule(quiet = TRUE) 107 | 108 | invoke(schedule, "p1", quiet = TRUE) 109 | 110 | expect_snapshot(schedule$get_status()[, c("invoked", "success")]) 111 | }) 112 | }) 113 | 114 | test_that("invoke only runs the pipeline selected", { 115 | 116 | withr::with_tempdir({ 117 | dir.create("pipelines") 118 | writeLines( 119 | " 120 | #' @maestroFrequency hourly 121 | pipe1 <- function() { 122 | 2 123 | } 124 | 125 | #' @maestroFrequency daily 126 | pipe2 <- function() { 127 | 4 128 | } 129 | ", 130 | con = "pipelines/invoked.R" 131 | ) 132 | 133 | schedule <- build_schedule(quiet = TRUE) 134 | 135 | invoke(schedule, "pipe2", quiet = TRUE) 136 | 137 | expect_equal(sum(schedule$get_status()$invoked), 1) 138 | }) 139 | }) -------------------------------------------------------------------------------- /R/invoke.R: -------------------------------------------------------------------------------- 1 | #' Manually run a pipeline regardless of schedule 2 | #' 3 | #' Instantly run a single pipeline from the schedule. This is useful for testing 4 | #' purposes or if you want to just run something one-off. 5 | #' 6 | #' Scheduling parameters such as the frequency, start time, and specifiers are ignored. 7 | #' The pipeline will be run even if `maestroSkip` is present. If the pipeline is a DAG 8 | #' pipeline, `invoke` will attempt to execute the full DAG. 9 | #' 10 | #' @inheritParams run_schedule 11 | #' @param pipe_name name of a single pipe name from the schedule 12 | #' @param ... other arguments passed to `run_schedule()` 13 | #' @param quiet silence metrics to the console (default = `FALSE`). Note this does not affect messages generated from pipelines when `log_to_console = TRUE`. 14 | #' @param log_to_console whether or not to include pipeline messages, warnings, errors to the console (default = `FALSE`) (see Logging & Console Output section) 15 | #' 16 | #' @return invisible 17 | #' @export 18 | #' 19 | #' @examples 20 | #' if (interactive()) { 21 | #' pipeline_dir <- tempdir() 22 | #' create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 23 | #' schedule <- build_schedule(pipeline_dir = pipeline_dir) 24 | #' 25 | #' invoke(schedule, "my_new_pipeline") 26 | #' } 27 | invoke <- function(schedule, pipe_name, resources = list(), ..., quiet = TRUE, log_to_console = FALSE) { 28 | 29 | if (!"MaestroSchedule" %in% class(schedule)) { 30 | cli::cli_abort( 31 | c("Schedule must be an object of {.cls MaestroSchedule} and not an object of class {.cls {class(schedule)}}.", 32 | "i" = "Use {.fn build_schedule} to create a valid schedule."), 33 | call = rlang::caller_env() 34 | ) 35 | } 36 | 37 | pipe_names <- schedule$PipelineList$get_pipe_names() 38 | 39 | if (!rlang::is_scalar_character(pipe_name)) { 40 | cli::cli_abort( 41 | c("`pipe_name` must be a single character string referencing the name of a pipeline in the schedule.", 42 | "i" = "Available pipe_names are {.pkg {sort(pipe_names)}}"), 43 | call = rlang::caller_env() 44 | ) 45 | } 46 | 47 | # Make sure pipe name is in the schedule 48 | if (!pipe_name %in% pipe_names) { 49 | cli::cli_abort( 50 | c("{.code {pipe_name}} is not the name of a pipeline in the schedule.", 51 | "i" = "Available pipe_names are {.pkg {sort(pipe_names)}}"), 52 | call = rlang::caller_env() 53 | ) 54 | } 55 | 56 | # Ensure that elements in resources are named 57 | if (length(resources) > 0) { 58 | resources_length <- length(resources) 59 | n_named <- sum(names(resources) != "") 60 | if (resources_length > n_named) { 61 | cli::cli_abort( 62 | "All elements in `resources` must be named." 63 | ) 64 | } 65 | 66 | n_uniq_names <- length(unique(names(resources))) 67 | if (resources_length > n_uniq_names) { 68 | cli::cli_abort( 69 | "All elements in `resources` must have unique names." 70 | ) 71 | } 72 | } 73 | 74 | tryCatch({ 75 | schedule$PipelineList$run( 76 | ..., 77 | resources = resources, 78 | pipes_to_run = pipe_name, 79 | quiet = quiet, 80 | log_to_console = log_to_console 81 | ) 82 | }, error = function(e) { 83 | 84 | if (e$message == "unused argument (`NA` = NULL)") { 85 | cli::cli_abort( 86 | c( 87 | "Failed to invoke pipeline {.code {pipe_name}}", 88 | "i" = "Did you forget to pass pipeline arguments as `resources = list(name = val)`?" 89 | ), 90 | call = NULL 91 | ) 92 | } else { 93 | cli::cli_abort( 94 | "Failed to invoke pipeline {.code {pipe_name}}", 95 | call = NULL 96 | ) 97 | } 98 | }) 99 | 100 | return(invisible()) 101 | } 102 | -------------------------------------------------------------------------------- /R/create_pipeline.R: -------------------------------------------------------------------------------- 1 | #' Create a new pipeline in a pipelines directory 2 | #' 3 | #' Allows the creation of new pipelines (R scripts) and fills in the maestro tags as specified. 4 | #' 5 | #' @param pipe_name name of the pipeline and function 6 | #' @param pipeline_dir directory containing the pipeline scripts 7 | #' @param frequency how often the pipeline should run (e.g., 1 day, daily, 3 hours, 4 months). Fills in maestroFrequency tag 8 | #' @param start_time start time of the pipeline schedule. Fills in maestroStartTime tag 9 | #' @param tz timezone that pipeline will be scheduled in. Fills in maestroTz tag 10 | #' @param log_level log level for the pipeline (e.g., INFO, WARN, ERROR). Fills in maestroLogLevel tag 11 | #' @param open whether or not to open the script upon creation 12 | #' @param quiet whether to silence messages in the console (default = `FALSE`) 13 | #' @param overwrite whether or not to overwrite an existing pipeline of the same name and location. 14 | #' @param skip whether to skip the pipeline when running in the orchestrator (default = `FALSE`) 15 | #' @param inputs vector of names of pipelines that input into this pipeline (default = `NULL` for no inputs) 16 | #' @param outputs vector of names of pipelines that receive output from this pipeline (default = `NULL` for no outputs) 17 | #' @param priority a single positive integer corresponding to the order in which this pipeline will be invoked in the presence of other simultaneously invoked pipelines. 18 | #' 19 | #' @return invisible 20 | #' @export 21 | #' @examples 22 | #' if (interactive()) { 23 | #' pipeline_dir <- tempdir() 24 | #' create_pipeline( 25 | #' "extract_data", 26 | #' pipeline_dir = pipeline_dir, 27 | #' frequency = "1 hour", 28 | #' open = FALSE, 29 | #' quiet = TRUE, 30 | #' overwrite = TRUE 31 | #' ) 32 | #' 33 | #' create_pipeline( 34 | #' "new_job", 35 | #' pipeline_dir = pipeline_dir, 36 | #' frequency = "20 minutes", 37 | #' start_time = as.POSIXct("2024-06-21 12:20:00"), 38 | #' log_level = "ERROR", 39 | #' open = FALSE, 40 | #' quiet = TRUE, 41 | #' overwrite = TRUE 42 | #' ) 43 | #' } 44 | create_pipeline <- function( 45 | pipe_name, 46 | pipeline_dir = "pipelines", 47 | frequency = "1 day", 48 | start_time = Sys.Date(), 49 | tz = "UTC", 50 | log_level = "INFO", 51 | quiet = FALSE, 52 | open = interactive(), 53 | overwrite = FALSE, 54 | skip = FALSE, 55 | inputs = NULL, 56 | outputs = NULL, 57 | priority = NULL 58 | ) { 59 | 60 | skip <- if (skip) { 61 | "\n#' @maestroSkip" 62 | } else { 63 | "" 64 | } 65 | 66 | inputs <- if (!is.null(inputs)) { 67 | paste("\n#'", paste(inputs, collapse = " ")) 68 | } else { 69 | "" 70 | } 71 | 72 | outputs <- if (!is.null(outputs)) { 73 | paste("\n#'", paste(outputs, collapse = " ")) 74 | } else { 75 | "" 76 | } 77 | 78 | priority <- if (!is.null(priority)) { 79 | if (!(rlang::is_scalar_integerish(priority) && priority > 0)) { 80 | cli::cli_abort( 81 | c("`priority` must be a single positive whole number."), 82 | call = NULL 83 | ) 84 | } 85 | paste("\n#' @maestroPriority", priority) 86 | } else { 87 | "" 88 | } 89 | 90 | # Makes a valid name for a pipe 91 | pipe_name <- gsub("\\.", "_", make.names(pipe_name)) 92 | 93 | script <- readLines(system.file("pipeline_template", package = "maestro")) |> 94 | paste(collapse = "\n") |> 95 | glue::glue( 96 | .open = "{{", 97 | .close = "}}", 98 | .null = NULL 99 | ) 100 | 101 | path <- file.path(pipeline_dir, paste0(pipe_name, ".R")) 102 | 103 | if (!dir.exists(pipeline_dir)) { 104 | dir.create(pipeline_dir) 105 | } 106 | 107 | if (file.exists(path)) { 108 | if (!overwrite) { 109 | cli::cli_abort( 110 | c("File {.file {path}} already exists.", 111 | "Set {.code create_pipeline(overwrite = TRUE)} to overwrite anyway."), 112 | call = NULL 113 | ) 114 | } else { 115 | if (!quiet) cli::cli_alert_warning("Overwriting existing pipeline at {.file {path}}.") 116 | } 117 | } 118 | 119 | writeLines( 120 | script, 121 | path 122 | ) 123 | 124 | if (open) { 125 | rstudioapi::documentOpen(path) 126 | } 127 | 128 | if(!quiet) { 129 | cli::cli_alert_success("Created pipeline at {.file {path}}") 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | # maestro maestro website 6 | 7 | 8 | 9 | [![R-CMD-check](https://github.com/whipson/maestro/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/whipson/maestro/actions/workflows/R-CMD-check.yaml) 10 | [![Codecov test 11 | coverage](https://codecov.io/gh/whipson/maestro/branch/main/graph/badge.svg)](https://app.codecov.io/gh/whipson/maestro?branch=main) 12 | [![CRAN_Status_Badge](https://www.r-pkg.org/badges/version/maestro)](https://cran.r-project.org/package=maestro) 13 | [![Total 14 | Downloads](https://cranlogs.r-pkg.org/badges/grand-total/maestro)](https://CRAN.R-project.org/package=maestro) 15 | [![Monthly 16 | Downloads](https://cranlogs.r-pkg.org/badges/maestro)](https://cran.r-project.org/package=maestro) 17 | 18 | 19 | `maestro` is a lightweight framework for creating and orchestrating data 20 | pipelines in R. At its core, maestro is an R script scheduler that is 21 | unique in two ways: 22 | 23 | 1. Stateless: It does not need to be continuously running - it can be 24 | run in a serverless architecture 25 | 2. Use of *rounded scheduling*: The timeliness of pipeline executions 26 | depends on how often you run your orchestrator 27 | 28 | In `maestro` you create **pipelines** (functions) and schedule them 29 | using `roxygen2` tags - these are special comments (decorators) above 30 | each function. Then you create an **orchestrator** containing `maestro` 31 | functions for scheduling and invoking the pipelines. 32 | 33 | ## Installation 34 | 35 | `maestro` is available on CRAN and can be installed via: 36 | 37 | ``` r 38 | install.packages("maestro") 39 | ``` 40 | 41 | Or, try out the development version via: 42 | 43 | ``` r 44 | devtools::install_github("https://github.com/whipson/maestro") 45 | ``` 46 | 47 | ## Project Setup 48 | 49 | A `maestro` project needs at least two components: 50 | 51 | 1. A collection of R pipelines (functions) that you want to schedule 52 | 2. A single orchestrator script that kicks off the scripts when they’re 53 | scheduled to run 54 | 55 | The project file structure will look like this: 56 | 57 | sample_project 58 | ├── orchestrator.R 59 | └── pipelines 60 | ├── my_etl.R 61 | ├── pipe1.R 62 | └── pipe2.R 63 | 64 | Use `maestro::create_maestro()` to easily create this project structure 65 | in a blank R project. 66 | 67 | Let’s look at each of these in more detail. 68 | 69 | ### Pipelines 70 | 71 | A pipeline is task we want to run. This task may involve retrieving data 72 | from a source, performing cleaning and computation on the data, then 73 | sending it to a destination. `maestro` is not concerned with what your 74 | pipeline does, but rather *when* you want to run it. Here’s a simple 75 | pipeline in `maestro`: 76 | 77 | ``` r 78 | #' Example ETL pipeline 79 | #' @maestroFrequency 1 day 80 | #' @maestroStartTime 2024-03-25 12:30:00 81 | my_etl <- function() { 82 | 83 | # Pretend we're getting data from a source 84 | message("Get data") 85 | extracted <- mtcars 86 | 87 | # Transform 88 | message("Transforming") 89 | transformed <- extracted |> 90 | dplyr::mutate(hp_deviation = hp - mean(hp)) 91 | 92 | # Load - write to a location 93 | message("Writing") 94 | # write.csv(transformed, file = paste0("transformed_mtcars_", Sys.Date(), ".csv")) 95 | } 96 | ``` 97 | 98 | What makes this a `maestro` pipeline is the use of special 99 | *roxygen*-style comments above the function definition: 100 | 101 | - `#' @maestroFrequency 1 day` indicates that this function should 102 | execute at a daily frequency. 103 | 104 | - `#' @maestroStartTime 2024-03-25 12:30:00` denotes the first time it 105 | should run. 106 | 107 | In other words, we’d expect it to run every day at 12:30 starting the 108 | 25th of March 2024. There are more `maestro` tags than these ones and 109 | all follow the camelCase convention established by `roxygen2`. 110 | 111 | ### Orchestrator 112 | 113 | The orchestrator is a script that checks the schedules of all the 114 | pipelines in a `maestro` project and executes them. The orchestrator 115 | also handles global execution tasks such as collecting logs and managing 116 | shared resources like global objects and custom functions. 117 | 118 | You have the option of using Quarto, RMarkdown, or a straight-up R 119 | script for the orchestrator, but the former two have some advantages 120 | with respect to deployment on Posit Connect. 121 | 122 | A simple orchestrator looks like this: 123 | 124 | ``` r 125 | library(maestro) 126 | 127 | # Look through the pipelines directory for maestro pipelines to create a schedule 128 | schedule <- build_schedule(pipeline_dir = "pipelines") 129 | 130 | # Checks which pipelines are due to run and then executes them 131 | output <- run_schedule( 132 | schedule, 133 | orch_frequency = "1 day" 134 | ) 135 | ``` 136 | 137 | 139 | 140 | The function `build_schedule()` scours through all the pipelines in the 141 | project and builds a schedule. Then `run_schedule()` checks each 142 | pipeline’s scheduled time against the system time within some margin of 143 | rounding and calls those pipelines to run. 144 | -------------------------------------------------------------------------------- /man/run_schedule.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/run_schedule.R 3 | \name{run_schedule} 4 | \alias{run_schedule} 5 | \title{Run a schedule} 6 | \usage{ 7 | run_schedule( 8 | schedule, 9 | orch_frequency = "1 day", 10 | check_datetime = lubridate::now(tzone = "UTC"), 11 | resources = list(), 12 | run_all = FALSE, 13 | n_show_next = 5, 14 | cores = 1, 15 | log_file_max_bytes = 1e+06, 16 | quiet = FALSE, 17 | log_to_console = FALSE, 18 | log_to_file = FALSE 19 | ) 20 | } 21 | \arguments{ 22 | \item{schedule}{object of type MaestroSchedule created using \code{build_schedule()}} 23 | 24 | \item{orch_frequency}{of the orchestrator, a single string formatted like "1 day", "2 weeks", "hourly", etc.} 25 | 26 | \item{check_datetime}{datetime against which to check the running of pipelines (default is current system time in UTC)} 27 | 28 | \item{resources}{named list of shared resources made available to pipelines as needed} 29 | 30 | \item{run_all}{run all pipelines regardless of the schedule (default is \code{FALSE}) - useful for testing. 31 | Does not apply to pipes with a \code{maestroSkip} tag. Conditional pipelines using \code{maestroRunIf} still behave according to their condition.} 32 | 33 | \item{n_show_next}{show the next n scheduled pipes} 34 | 35 | \item{cores}{number of cpu cores to run if running in parallel. If > 1, \code{furrr} is used and a multisession plan must be executed in the orchestrator (see details)} 36 | 37 | \item{log_file_max_bytes}{numeric specifying the maximum number of bytes allowed in the log file before purging the log (within a margin of error)} 38 | 39 | \item{quiet}{silence metrics to the console (default = \code{FALSE}). Note this does not affect messages generated from pipelines when \code{log_to_console = TRUE}.} 40 | 41 | \item{log_to_console}{whether or not to include pipeline messages, warnings, errors to the console (default = \code{FALSE}) (see Logging & Console Output section)} 42 | 43 | \item{log_to_file}{either a boolean to indicate whether to create and append to a \code{maestro.log} or a character path to a specific log file. If \code{FALSE} or \code{NULL} it will not log to a file.} 44 | } 45 | \value{ 46 | MaestroSchedule object 47 | } 48 | \description{ 49 | Given a schedule in a \code{maestro} project, runs the pipelines that are scheduled to execute 50 | based on the current time. 51 | } 52 | \details{ 53 | \subsection{Pipeline schedule logic}{ 54 | 55 | The function \code{run_schedule()} examines each pipeline in the schedule table and determines 56 | whether it is scheduled to run at the current time using some simple time arithmetic. We assume 57 | \code{run_schedule(schedule, check_datetime = Sys.time())}, but this need not be the case. 58 | } 59 | 60 | \subsection{Output}{ 61 | 62 | \code{run_schedule()} returns the same MaestroSchedule object with modified attributes. Use \code{get_status()} 63 | to examine the status of each pipeline and use \code{get_artifacts()} to get any return values from the 64 | pipelines as a list. 65 | } 66 | 67 | \subsection{Pipelines with arguments (resources)}{ 68 | 69 | If a pipeline takes an argument that doesn't include a default value, these can be supplied 70 | in the orchestrator via \code{run_schedule(resources = list(arg1 = val))}. The name of the argument 71 | used by the pipeline must match the name of the argument in the list. Currently, each named 72 | resource must refer to a single object. In other words, you can't have two pipes using 73 | the same argument but requiring different values. 74 | } 75 | 76 | \subsection{Running in parallel}{ 77 | 78 | Pipelines can be run in parallel using the \code{cores} argument. First, you must run \code{future::plan(future::multisession)} 79 | in the orchestrator. Then, supply the desired number of cores to the \code{cores} argument. Note that 80 | console output appears different in multicore mode. 81 | } 82 | 83 | \subsection{Logging & Console Output}{ 84 | 85 | By default, \code{maestro} suppresses pipeline messages, warnings, and errors from appearing in the console, but 86 | messages coming from \code{print()} and other console logging packages like \code{cli} and \code{logger} are not suppressed 87 | and will be interwoven into the output generated from \code{run_schedule()}. Messages from \code{cat()} and related functions are always suppressed 88 | due to the nature of how those functions operate with standard output. 89 | 90 | Users are advised to make use of R's \code{message()}, \code{warning()}, and \code{stop()} functions in their pipelines 91 | for managing conditions. Use \code{log_to_console = TRUE} to print these to the console. 92 | 93 | Maestro can generate a log file that is appended to each time the orchestrator is run. Use \code{log_to_file = TRUE} or \code{log_to_file = '[path-to-file]'} and 94 | maestro will create/append to a file in the project directory. 95 | This log file will be appended to until it exceeds the byte size defined in \code{log_file_max_bytes} argument after which 96 | the log file is deleted. 97 | } 98 | } 99 | \examples{ 100 | 101 | if (interactive()) { 102 | pipeline_dir <- tempdir() 103 | create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 104 | schedule <- build_schedule(pipeline_dir = pipeline_dir) 105 | 106 | # Runs the schedule every 1 day 107 | run_schedule( 108 | schedule, 109 | orch_frequency = "1 day", 110 | quiet = TRUE 111 | ) 112 | 113 | # Runs the schedule every 15 minutes 114 | run_schedule( 115 | schedule, 116 | orch_frequency = "15 minutes", 117 | quiet = TRUE 118 | ) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /README.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | output: 3 | github_document 4 | format: gfm 5 | always_allow_html: yes 6 | default-image-extension: "" 7 | --- 8 | 9 | 10 | 11 | ```{r, include = FALSE, cache = FALSE} 12 | #| echo: false 13 | knitr::opts_chunk$set( 14 | fig.path = "man/figures/README-", 15 | out.width = "100%" 16 | ) 17 | asciicast::init_knitr_engine( 18 | echo = TRUE, 19 | echo_input = FALSE 20 | ) 21 | options(asciicast_theme = "pkgdown") 22 | ``` 23 | 24 | # maestro maestro website 25 | 26 | 27 | 28 | [![R-CMD-check](https://github.com/whipson/maestro/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/whipson/maestro/actions/workflows/R-CMD-check.yaml) 29 | [![Codecov test coverage](https://codecov.io/gh/whipson/maestro/branch/main/graph/badge.svg)](https://app.codecov.io/gh/whipson/maestro?branch=main) 30 | [![CRAN\_Status\_Badge](https://www.r-pkg.org/badges/version/maestro)](https://cran.r-project.org/package=maestro) 31 | [![Total Downloads](https://cranlogs.r-pkg.org/badges/grand-total/maestro)](https://CRAN.R-project.org/package=maestro) 32 | [![Monthly Downloads](https://cranlogs.r-pkg.org/badges/maestro)](https://cran.r-project.org/package=maestro) 33 | 34 | 35 | `maestro` is a lightweight framework for creating and orchestrating data pipelines in R. At its core, maestro is an R script scheduler that is unique in two ways: 36 | 37 | 1. Stateless: It does not need to be continuously running - it can be run in a serverless architecture 38 | 2. Use of *rounded scheduling*: The timeliness of pipeline executions depends on how often you run your orchestrator 39 | 40 | In `maestro` you create **pipelines** (functions) and schedule them using `roxygen2` tags - these are special comments (decorators) above each function. Then you create an **orchestrator** containing `maestro` functions for scheduling and invoking the pipelines. 41 | 42 | ## Installation 43 | 44 | `maestro` is available on CRAN and can be installed via: 45 | 46 | ```r 47 | install.packages("maestro") 48 | ``` 49 | Or, try out the development version via: 50 | 51 | ``` r 52 | devtools::install_github("https://github.com/whipson/maestro") 53 | ``` 54 | 55 | ## Project Setup 56 | 57 | A `maestro` project needs at least two components: 58 | 59 | 1. A collection of R pipelines (functions) that you want to schedule 60 | 2. A single orchestrator script that kicks off the scripts when they're scheduled to run 61 | 62 | The project file structure will look like this: 63 | 64 | ``` 65 | sample_project 66 | ├── orchestrator.R 67 | └── pipelines 68 | ├── my_etl.R 69 | ├── pipe1.R 70 | └── pipe2.R 71 | ``` 72 | 73 | Use `maestro::create_maestro()` to easily create this project structure in a blank R project. 74 | 75 | Let's look at each of these in more detail. 76 | 77 | ### Pipelines 78 | 79 | A pipeline is task we want to run. This task may involve retrieving data from a source, performing cleaning and computation on the data, then sending it to a destination. `maestro` is not concerned with what your pipeline does, but rather *when* you want to run it. Here's a simple pipeline in `maestro`: 80 | 81 | ```{r} 82 | #' Example ETL pipeline 83 | #' @maestroFrequency 1 day 84 | #' @maestroStartTime 2024-03-25 12:30:00 85 | my_etl <- function() { 86 | 87 | # Pretend we're getting data from a source 88 | message("Get data") 89 | extracted <- mtcars 90 | 91 | # Transform 92 | message("Transforming") 93 | transformed <- extracted |> 94 | dplyr::mutate(hp_deviation = hp - mean(hp)) 95 | 96 | # Load - write to a location 97 | message("Writing") 98 | # write.csv(transformed, file = paste0("transformed_mtcars_", Sys.Date(), ".csv")) 99 | } 100 | ``` 101 | 102 | What makes this a `maestro` pipeline is the use of special *roxygen*-style comments above the function definition: 103 | 104 | - `#' @maestroFrequency 1 day` indicates that this function should execute at a daily frequency. 105 | 106 | - `#' @maestroStartTime 2024-03-25 12:30:00` denotes the first time it should run. 107 | 108 | In other words, we'd expect it to run every day at 12:30 starting the 25th of March 2024. There are more `maestro` tags than these ones and all follow the camelCase convention established by `roxygen2`. 109 | 110 | ### Orchestrator 111 | 112 | The orchestrator is a script that checks the schedules of all the pipelines in a `maestro` project and executes them. The orchestrator also handles global execution tasks such as collecting logs and managing shared resources like global objects and custom functions. 113 | 114 | You have the option of using Quarto, RMarkdown, or a straight-up R script for the orchestrator, but the former two have some advantages with respect to deployment on Posit Connect. 115 | 116 | A simple orchestrator looks like this: 117 | 118 | ```{asciicast} 119 | library(maestro) 120 | 121 | # Look through the pipelines directory for maestro pipelines to create a schedule 122 | schedule <- build_schedule(pipeline_dir = "pipelines") 123 | 124 | # Checks which pipelines are due to run and then executes them 125 | output <- run_schedule( 126 | schedule, 127 | orch_frequency = "1 day" 128 | ) 129 | ``` 130 | 131 | The function `build_schedule()` scours through all the pipelines in the project and builds a schedule. Then `run_schedule()` checks each pipeline's scheduled time against the system time within some margin of rounding and calls those pipelines to run. 132 | -------------------------------------------------------------------------------- /tests/testthat/test-build_schedule_entry.R: -------------------------------------------------------------------------------- 1 | test_that("can create a schedule entry from a single well-documented fun", { 2 | res <- build_schedule_entry( 3 | test_path("test_pipelines/test_pipeline_daily_good.R") 4 | ) 5 | expect_s3_class(res, "MaestroPipelineList") 6 | }) 7 | 8 | test_that("can create a schedule entry from a default tagged fun", { 9 | res <- build_schedule_entry( 10 | test_path("test_pipelines/test_pipeline_daily_default.R") 11 | ) 12 | expect_s3_class(res, "MaestroPipelineList") 13 | expect_equal(length(res$MaestroPipelines), 1) 14 | }) 15 | 16 | test_that("invalid tags trigger error", { 17 | build_schedule_entry( 18 | test_path("test_pipelines/test_pipeline_daily_bad.R") 19 | ) |> 20 | expect_error(regexp = "Invalid maestroFrequency") 21 | }) |> 22 | suppressMessages() 23 | 24 | test_that("can create a schedule entry from a multi-function script", { 25 | res <- build_schedule_entry( 26 | test_path("test_pipelines/test_multi_fun_pipeline.R") 27 | ) 28 | expect_s3_class(res, "MaestroPipelineList") 29 | expect_equal(length(res$MaestroPipelines), 2) 30 | }) |> 31 | suppressMessages() 32 | 33 | test_that("Errors on a pipeline with no tagged functions", { 34 | build_schedule_entry( 35 | test_path("test_pipelines_parse_all_bad/test_pipeline_no_func.R") 36 | ) |> 37 | expect_error(regexp = "tags present in") 38 | }) |> 39 | suppressMessages() 40 | 41 | test_that("Errors on pipeline that doesn't use functions and returns null", { 42 | 43 | expect_error({ 44 | res <- build_schedule_entry( 45 | test_path("test_pipelines_parse_all_bad/tagged_but_no_func.R") 46 | ) 47 | }, regexp = "has tags but no function") 48 | }) |> 49 | suppressMessages() 50 | 51 | test_that("Errors on pipeline with isolated tags", { 52 | expect_error({ 53 | res <- build_schedule_entry( 54 | test_path("test_pipelines_parse_all_bad/tagged_but_no_func_multi.R") 55 | ) 56 | }, regexp = "has tags but no function") 57 | }) 58 | 59 | test_that("Informative error on parsing schedule entry with an error", { 60 | expect_error({ 61 | res <- build_schedule_entry( 62 | test_path("test_pipelines_parse_all_bad/test_pipeline_script_with_error.R") 63 | ) 64 | }, regexp = "Could not build") 65 | }) 66 | 67 | test_that("Script with untagged function isn't treated as a scheduled pipeline", { 68 | res <- build_schedule_entry( 69 | test_path("test_pipelines_parse_all_good/pipe_with_custom_fun.R") 70 | ) 71 | expect_equal(length(res$MaestroPipelines), 1) 72 | }) 73 | 74 | test_that("Script with good specifiers parses well", { 75 | expect_no_error({ 76 | schedule <- build_schedule_entry( 77 | test_path("test_pipelines_parse_all_good/specifiers.R") 78 | ) 79 | }) 80 | }) 81 | 82 | test_that("Script with bad specifiers errors", { 83 | schedule <- build_schedule_entry( 84 | test_path("test_pipelines_parse_all_bad/specifiers.R") 85 | ) |> 86 | expect_error(regexp = "pipeline must have") 87 | }) 88 | 89 | test_that("Pipeline with inputs checks for .input", { 90 | schedule <- build_schedule_entry( 91 | test_path("test_pipelines_parse_all_bad/dags.R") 92 | ) |> 93 | expect_error(regexp = "pipeline must have a parameter") 94 | 95 | expect_no_error({ 96 | schedule <- build_schedule_entry( 97 | test_path("test_pipelines_dags_good/dags.R") 98 | ) 99 | }) 100 | }) 101 | 102 | test_that("Errors if pipeline self-references as input", { 103 | schedule <- build_schedule_entry( 104 | test_path("test_pipelines_parse_all_bad/dags_self_reference_inputs.R") 105 | ) |> 106 | expect_error(regexp = "cannot contain self-references") 107 | }) 108 | 109 | test_that("Errors if pipeline self-references as output", { 110 | schedule <- build_schedule_entry( 111 | test_path("test_pipelines_parse_all_bad/dags_self_reference_outputs.R") 112 | ) |> 113 | expect_error(regexp = "cannot contain self-references") 114 | }) 115 | 116 | test_that("Pipelines with maestro tag get picked up", { 117 | res <- build_schedule_entry( 118 | test_path("test_pipelines/test_pipeline_maestro.R") 119 | ) 120 | expect_equal(length(res$MaestroPipelines), 1) 121 | }) 122 | 123 | test_that("Pipelines with maestroHours and maestroFrequency = 1 hour are valid", { 124 | 125 | withr::with_tempdir({ 126 | dir.create("pipelines") 127 | writeLines( 128 | " 129 | #' @maestroTz America/Halifax 130 | #' @maestroFrequency 1 hour 131 | #' @maestroStartTime 2025-01-01 00:00:00 132 | #' @maestroHours 0 4 7 12 18 133 | hourly <- function() { 134 | 135 | } 136 | ", 137 | con = "pipelines/hourly.R" 138 | ) 139 | 140 | expect_no_error({ 141 | build_schedule_entry("pipelines/hourly.R") 142 | }) 143 | }) 144 | }) 145 | 146 | test_that("Pipelines with maestroStartTime in HH:MM:SS and a multi-unit frequency fail", { 147 | 148 | withr::with_tempdir({ 149 | dir.create("pipelines") 150 | writeLines( 151 | " 152 | #' @maestroTz America/Halifax 153 | #' @maestroFrequency 2 days 154 | #' @maestroStartTime 00:00:00 155 | hourly <- function() { 156 | 157 | } 158 | ", 159 | con = "pipelines/hourly.R" 160 | ) 161 | 162 | expect_error({ 163 | build_schedule_entry("pipelines/hourly.R") 164 | }, regexp = "Cannot use a `@maestroStartTime`") 165 | }) 166 | 167 | withr::with_tempdir({ 168 | dir.create("pipelines") 169 | writeLines( 170 | " 171 | #' @maestroTz America/Halifax 172 | #' @maestroFrequency weekly 173 | #' @maestroStartTime 00:00:00 174 | hourly <- function() { 175 | 176 | } 177 | ", 178 | con = "pipelines/hourly.R" 179 | ) 180 | 181 | expect_error({ 182 | build_schedule_entry("pipelines/hourly.R") 183 | }, regexp = "`@maestroStartTime`") 184 | }) 185 | }) 186 | -------------------------------------------------------------------------------- /vignettes/maestro-4-directed-acyclic-graphs.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Directed Acyclic Graphs (DAGs)" 3 | output: rmarkdown::html_vignette 4 | vignette: > 5 | %\VignetteIndexEntry{Directed Acyclic Graphs (DAGs)} 6 | %\VignetteEngine{knitr::rmarkdown} 7 | %\VignetteEncoding{UTF-8} 8 | --- 9 | ```{r echo=FALSE} 10 | knitr::opts_chunk$set( 11 | collapse = FALSE, 12 | comment = "", 13 | out.width = "100%", 14 | cache = FALSE, 15 | asciicast_knitr_output = "html" 16 | ) 17 | 18 | asciicast::init_knitr_engine( 19 | echo = TRUE, 20 | echo_input = FALSE, 21 | same_process = TRUE, 22 | startup = quote({ 23 | library(maestro) 24 | set.seed(1) 25 | }) 26 | ) 27 | options(asciicast_theme = "pkgdown") 28 | ``` 29 | 30 | 31 | A [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) or DAG is a kind of network graph where nodes are connected by edges, and these connections cannot loop back or cycle. Most data orchestration tools lay out a data pipeline as a DAG where data is passed from one function to the next until it reaches the end. This allows for more module, single-purpose functions and can make it easier to identify where errors are occurring. 32 | 33 | You can create DAG pipelines in maestro using the `maestroInputs` and/or `maestroOutputs` tags. Let's see this in action. 34 | 35 | ```{r echo=FALSE, warning=FALSE, message=FALSE} 36 | invisible(dir.create("pipelines")) 37 | writeLines( 38 | " 39 | #' @maestroOutputs high_road low_road 40 | start <- function() { 41 | c('a', 'A') 42 | } 43 | 44 | #' @maestroInputs start 45 | high_road <- function(.input) { 46 | toupper(.input) 47 | } 48 | 49 | #' @maestroInputs start 50 | low_road <- function(.input) { 51 | tolower(.input) 52 | }", 53 | con = "pipelines/dags.R" 54 | ) 55 | ``` 56 | 57 | We'll create three simple pipelines. `start` outputs a vector, `high_road` takes an input and makes it all uppercase, `low_road` makes the input all lowercase. We use the `maestroOutputs` tag to indicate the names of the downstream pipelines (i.e., these pipelines use the output of the target pipeline as input) and we use the `maestroInputs` tag to indicate the names of pipelines that are used as input.[^1] 58 | 59 | [^1]: Specifying the outputs and inputs is redundant. You can specify just the outputs or just the inputs if you like, but make sure all pipelines are identified as maestro pipelines by including at least one maestro tag (you could make use of the catch-all `@maestro` tag for this. 60 | 61 | Note the use of `.input` as a parameter for all pipelines that receive an input. It is important to have this here to enable the passing of data from inputs to outputs. It must be named `.input`. 62 | 63 | ```{r eval=FALSE} 64 | #' ./pipelines/dags.R 65 | #' @maestroOutputs high_road low_road 66 | start <- function() { 67 | c('a', 'A') 68 | } 69 | 70 | #' @maestroInputs start 71 | high_road <- function(.input) { 72 | toupper(.input) 73 | } 74 | 75 | #' @maestroInputs start 76 | low_road <- function(.input) { 77 | tolower(.input) 78 | } 79 | ``` 80 | 81 | Now we'll create and run the schedule. Notice that the output in the console will reflect the network structure of the DAG. 82 | 83 | ```{asciicast} 84 | # ./orchestrator.R 85 | library(maestro) 86 | 87 | schedule <- build_schedule(quiet = TRUE) 88 | 89 | status <- run_schedule( 90 | schedule, 91 | run_all = TRUE 92 | ) 93 | 94 | get_artifacts(schedule) 95 | ``` 96 | 97 | ## ETL Example 98 | 99 | A great case for using DAGs is with ETL/ELT pipelines. Each component of extract, transform, and load could be a single element in the DAG. Consider the example on the home page: 100 | 101 | ```{r eval=FALSE} 102 | #' Example ETL pipeline 103 | #' @maestroFrequency 1 day 104 | #' @maestroStartTime 2024-03-25 12:30:00 105 | my_etl <- function() { 106 | 107 | # Pretend we're getting data from a source 108 | message("Get data") 109 | extracted <- mtcars 110 | 111 | # Transform 112 | message("Transforming") 113 | transformed <- extracted |> 114 | dplyr::mutate(hp_deviation = hp - mean(hp)) 115 | 116 | # Load - write to a location 117 | message("Writing") 118 | write.csv(transformed, file = paste0("transformed_mtcars_", Sys.Date(), ".csv")) 119 | } 120 | ``` 121 | 122 | It's pretty concise, so we probably wouldn't bother breaking it apart in practice, but let's do it for illustrative purposes (and also get rid of the messaging). 123 | 124 | ```{r echo=FALSE, warning=FALSE, message=FALSE} 125 | invisible(file.remove("pipelines/dags.R")) 126 | invisible(dir.create("pipelines")) 127 | writeLines( 128 | " 129 | #' @maestroFrequency 1 day 130 | #' @maestroStartTime 2024-03-25 12:30:00 131 | #' @maestroOutputs transform 132 | extract <- function() { 133 | # Imagine this is something way more complicated, like a database call 134 | mtcars 135 | } 136 | 137 | #' @maestroOutputs load 138 | transform <- function(.input) { 139 | .input |> 140 | dplyr::mutate(hp_deviation = hp - mean(hp)) 141 | } 142 | 143 | #' @maestro 144 | load <- function(.input) { 145 | write.csv(.input, file = paste0('transformed_mtcars.csv')) 146 | }", 147 | con = "pipelines/etl.R" 148 | ) 149 | ``` 150 | 151 | ```{r eval=FALSE} 152 | #' @maestroFrequency 1 day 153 | #' @maestroStartTime 2024-03-25 12:30:00 154 | #' @maestroOutputs transform 155 | extract <- function() { 156 | # Imagine this is something way more complicated, like a database call 157 | mtcars 158 | } 159 | 160 | #' @maestroOutputs load 161 | transform <- function(.input) { 162 | .input |> 163 | dplyr::mutate(hp_deviation = hp - mean(hp)) 164 | } 165 | 166 | #' @maestro 167 | load <- function(.input) { 168 | write.csv(.input, file = paste0("transformed_mtcars.csv")) 169 | } 170 | ``` 171 | 172 | ```{asciicast} 173 | library(maestro) 174 | 175 | schedule <- build_schedule(quiet = TRUE) 176 | 177 | status <- run_schedule( 178 | schedule, 179 | run_all = TRUE 180 | ) 181 | ``` 182 | 183 | When developing these pipelines, it is helpful to visualize the dependency structure. We can do this by calling `show_network()` on the schedule: 184 | 185 | ```{r echo=FALSE, warning=FALSE, message=FALSE} 186 | library(maestro) 187 | 188 | schedule <- build_schedule(quiet = TRUE) 189 | 190 | show_network(schedule) 191 | ``` 192 | 193 | ```{r cleanup, echo=FALSE, message=FALSE, warning=FALSE} 194 | invisible(unlink("pipelines", recursive = TRUE)) 195 | invisible(unlink("transformed_mtcars.csv")) 196 | ``` 197 | -------------------------------------------------------------------------------- /vignettes/maestro-1-quick-start.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Quick Start" 3 | output: rmarkdown::html_vignette 4 | vignette: > 5 | %\VignetteIndexEntry{Quick Start} 6 | %\VignetteEngine{knitr::rmarkdown} 7 | %\VignetteEncoding{UTF-8} 8 | --- 9 | 10 | ```{r} 11 | #| echo: false 12 | 13 | knitr::opts_chunk$set( 14 | collapse = FALSE, 15 | comment = "", 16 | out.width = "100%", 17 | cache = FALSE, 18 | asciicast_knitr_output = "html" 19 | ) 20 | 21 | asciicast::init_knitr_engine( 22 | echo = TRUE, 23 | echo_input = FALSE, 24 | same_process = TRUE, 25 | startup = quote({ 26 | library(maestro) 27 | set.seed(1) 28 | }) 29 | ) 30 | 31 | options(asciicast_theme = "pkgdown") 32 | ``` 33 | 34 | 35 | A common task in data engineering is to automate, schedule, and monitor multiple data processing pipelines. This is called **orchestration**. `maestro` is an R package that helps orchestrate data pipelines. 36 | 37 | A fully realized `maestro` project involves the following components and actions: 38 | 39 | 1. Collection of **pipelines** (R functions to be orchestrated, such as batch ETL jobs) 40 | 41 | 2. **Orchestrator** - an R script or Quarto doc that orchestrates the pipelines and monitors them 42 | 43 | 3. A process external to R to schedule the orchestrator (e.g., cron, Posit Connect). 44 | 45 | ### Project Setup 46 | 47 | ```{r} 48 | library(maestro) 49 | ``` 50 | 51 | Create a `maestro` project in an existing project or a new project using `create_maestro()` or the New Project wizard in RStudio. This creates the orchestrator script and the folder of pipelines with one sample pipeline. Your project should look something like this: 52 | 53 | ```{r echo=FALSE, error=FALSE, warning=FALSE, message=FALSE} 54 | dir.create("pipelines") 55 | writeLines( 56 | " 57 | #' my_pipe maestro pipeline 58 | #' 59 | #' @maestroFrequency 1 day 60 | #' @maestroStartTime 2024-05-24 61 | #' @maestroTz UTC 62 | #' @maestroLogLevel INFO 63 | 64 | my_pipe <- function() { 65 | 66 | # Pipeline code 67 | }", 68 | con = "pipelines/my_pipe.R" 69 | ) 70 | ``` 71 | 72 | ``` 73 | maestro_project 74 | ├── maestro_project.Rproj 75 | ├── orchestrator.R 76 | └── pipelines 77 | ├── my_pipe.R 78 | └── another_pipe.R 79 | ``` 80 | 81 | ## Pipelines 82 | 83 | Pipelines are the jobs you want to automate, schedule, and monitor. For the most part, they're regular R functions with a special sprinkling of comments. 84 | 85 | ### Anatomy of a Pipeline 86 | 87 | A pipeline is simply an R function with decorators called maestro tags. Maestro tags are special code comments used for communicating the scheduling and configuration of a pipeline to the orchestrator. Let's take a quick look at the sample `my_pipe.R`: 88 | 89 | ```{r eval=FALSE} 90 | #' my_pipe maestro pipeline 91 | #' 92 | #' @maestroFrequency 1 day 93 | #' @maestroStartTime 2024-05-24 94 | #' @maestroTz UTC 95 | #' @maestroLogLevel INFO 96 | 97 | my_pipe <- function() { 98 | 99 | # Pipeline code 100 | } 101 | ``` 102 | 103 | `my_pipe` is a function with an empty body - so right now it won't do anything. The comments above are interpreted by `maestro` as "this function is scheduled to run every day starting at 2024-05-24 (00:00:00) UTC time". 104 | 105 | maestroFrequency and maestroStartTime are the most important tags for scheduling. Frequency is how often you want the pipeline to run and can be formatted as a single string like hourly, daily, weekly, biweekly, etc. or with a number and a unit (e.g., 1 day, 3 hours, etc.). 106 | 107 | Note that you don't need to provide all these tags. A single maestro tag is enough to distinguish it as a pipeline. Pipelines missing tags will use consistent defaults (e.g., if `maestroFrequency` is missing the default is 1 day/daily). 108 | 109 | In most use cases, the actual code inside of `my_pipe` would be to run an ETL job (extract data from a source, transform it, and load it into a file system or database). In technical terms, it's the *side effect* of the code and not its return value that is important. 110 | 111 | Here's a more realistic, albeit impractical, example: 112 | 113 | ```{r} 114 | #' my_pipe maestro pipeline 115 | #' 116 | #' @maestroFrequency 1 day 117 | #' @maestroStartTime 2024-05-24 118 | #' @maestroTz UTC 119 | #' @maestroLogLevel INFO 120 | 121 | my_pipe <- function() { 122 | 123 | random_data <- data.frame( 124 | letters = sample(letters, 10), 125 | numbers = sample.int(10) 126 | ) 127 | 128 | write.csv(random_data, file = tempfile()) 129 | } 130 | ``` 131 | 132 | ### Adding New Pipelines 133 | 134 | A project with a single pipeline is ok, but in `maestro` is more useful when you have multiple jobs to run. You can add more pipelines to your pipelines directory manually or use `create_pipeline()`: 135 | 136 | ```{r example-pipe} 137 | #| eval: false 138 | create_pipeline( 139 | pipe_name = "another_pipeline", 140 | pipeline_dir = "pipelines", 141 | frequency = "1 hour", 142 | start_time = "2024-05-17 15:00:00", 143 | tz = "America/Halifax", 144 | log_level = "ERROR" 145 | ) 146 | ``` 147 | 148 | ## Orchestrator 149 | 150 | The orchestrator is the process that schedules and monitors the pipelines. 151 | 152 | ### Anatomy of the Orchestrator 153 | 154 | The orchestrator can be an R script, Quarto/RMarkdown doc, but here we'll use a regular R script. Here is where you'll run `maestro` functions. The two main functions are `build_schedule()` and `run_schedule()`. 155 | 156 | ```{asciicast} 157 | library(maestro) 158 | 159 | schedule <- build_schedule() 160 | 161 | output <- run_schedule( 162 | schedule, 163 | orch_frequency = "1 hour" 164 | ) 165 | ``` 166 | 167 | Building the schedule gets `maestro` to look through the pipelines in the pipelines folder and creates a schedule object. Then, you pass that to `run_schedule()` along with how often the orchestrator is supposed to run. It is important to tell `maestro` how often it'll be checking the pipelines using the `orch_frequency` parameter. Here, we're informing it that the orchestrator is running every 1 hour. 168 | 169 | Importantly, it isn't `maestro`'s job to actually run it this often - it's *your* job to make sure it runs at that frequency (e.g., deploying it via cron or some cloud environment where code can be scheduled).[^1] 170 | 171 | [^1]: The decision around how often to run the orchestrator depends on the frequencies of the pipelines in the project. The simplest guideline is to take the frequency of your most often recurring pipeline and split that in half. So for example, if my most often running pipeline runs every 1 day then my orchestrator should run every 12 hours. 172 | 173 | ```{r cleanup, echo=FALSE, message=FALSE, warning=FALSE} 174 | unlink("pipelines", recursive = TRUE) 175 | ``` 176 | -------------------------------------------------------------------------------- /man/figures/README-/unnamed-chunk-3.svg: -------------------------------------------------------------------------------- 1 | ──[2025-08-0713:45:28]ℹ3scriptssuccessfullyparsedRunningpipelines▶✔my_etl[72ms]✔get_mtcars[43ms]✔multi_rng[65ms]Pipelineexecutioncompleted■|0.201secelapsed✔3successes|!1warning|✖0errors|◼3total!Use`last_run_warnings()`toshowpipelinewarnings.──────────────────────────────────────────────────────────────────────────────────Nextscheduledpipelines❯Pipename|Nextscheduledrun•my_etl|2025-08-09•get_mtcars|2025-08-09•multi_rng|2025-08-09 -------------------------------------------------------------------------------- /tests/testthat/test-conditionals.R: -------------------------------------------------------------------------------- 1 | test_that("Conditional pipes work with DAG pipelines via .input", { 2 | withr::with_tempdir({ 3 | dir.create("pipelines") 4 | writeLines( 5 | " 6 | #' @maestroFrequency 1 day 7 | #' @maestroOutputs p2 8 | p1 <- function() { 9 | TRUE 10 | } 11 | 12 | #' @maestroRunIf .input 13 | p2 <- function() { 14 | TRUE 15 | } 16 | ", 17 | con = "pipelines/conditionals.R" 18 | ) 19 | 20 | schedule <- build_schedule(quiet = TRUE) 21 | run_schedule( 22 | schedule, 23 | orch_frequency = "1 day", 24 | quiet = FALSE 25 | ) 26 | status <- get_status(schedule) 27 | }) 28 | 29 | expect_snapshot(status$invoked) 30 | 31 | withr::with_tempdir({ 32 | dir.create("pipelines") 33 | writeLines( 34 | " 35 | #' @maestroFrequency 1 day 36 | #' @maestroOutputs p2 37 | p1 <- function() { 38 | FALSE 39 | } 40 | 41 | #' @maestroRunIf .input 42 | p2 <- function() { 43 | TRUE 44 | } 45 | ", 46 | con = "pipelines/conditionals.R" 47 | ) 48 | 49 | schedule <- build_schedule(quiet = TRUE) 50 | run_schedule( 51 | schedule, 52 | orch_frequency = "1 day", 53 | quiet = TRUE 54 | ) 55 | status <- get_status(schedule) 56 | }) 57 | 58 | expect_snapshot(status$invoked) 59 | 60 | # More complex multiline eval 61 | withr::with_tempdir({ 62 | dir.create("pipelines") 63 | writeLines( 64 | " 65 | #' @maestroFrequency 1 day 66 | #' @maestroOutputs p2 67 | p1 <- function() { 68 | 7 69 | } 70 | 71 | #' @maestroRunIf 72 | #' is_mod <- (.input %% 2) == 0 73 | #' (is_mod * 4) == 4 74 | p2 <- function() { 75 | TRUE 76 | } 77 | ", 78 | con = "pipelines/conditionals.R" 79 | ) 80 | 81 | schedule <- build_schedule(quiet = TRUE) 82 | run_schedule( 83 | schedule, 84 | orch_frequency = "1 day", 85 | quiet = TRUE 86 | ) 87 | status <- get_status(schedule) 88 | }) 89 | 90 | expect_snapshot(status$invoked) 91 | }) 92 | 93 | test_that("DAGs with a conditional pipe in the middle by default halt further execution of the DAG", { 94 | 95 | withr::with_tempdir({ 96 | dir.create("pipelines") 97 | writeLines( 98 | " 99 | #' @maestroFrequency 1 day 100 | #' @maestroOutputs p2 101 | p1 <- function() { 102 | FALSE 103 | } 104 | 105 | #' @maestroRunIf .input 106 | p2 <- function() { 107 | TRUE 108 | } 109 | 110 | #' @maestroInputs p2 111 | p3 <- function(.input) { 112 | TRUE 113 | } 114 | ", 115 | con = "pipelines/conditionals.R" 116 | ) 117 | 118 | schedule <- build_schedule(quiet = TRUE) 119 | run_schedule( 120 | schedule, 121 | orch_frequency = "1 day", 122 | quiet = TRUE 123 | ) 124 | status <- get_status(schedule) 125 | }) 126 | 127 | expect_snapshot(status$invoked) 128 | }) 129 | 130 | test_that("Conditional pipes work using resources", { 131 | withr::with_tempdir({ 132 | dir.create("pipelines") 133 | writeLines( 134 | " 135 | #' @maestroFrequency 1 day 136 | #' @maestroRunIf var == 4 137 | p1 <- function() { 138 | TRUE 139 | } 140 | 141 | #' @maestroFrequency 1 day 142 | #' @maestroRunIf var == 3 143 | p2 <- function() { 144 | TRUE 145 | } 146 | ", 147 | con = "pipelines/conditionals.R" 148 | ) 149 | 150 | schedule <- build_schedule(quiet = TRUE) 151 | run_schedule( 152 | schedule, 153 | orch_frequency = "1 day", 154 | quiet = TRUE, 155 | resources = list(var = 4) 156 | ) 157 | status <- get_status(schedule) 158 | }) 159 | 160 | expect_snapshot(status$invoked) 161 | }) 162 | 163 | test_that("Conditions with errors are handled", { 164 | withr::with_tempdir({ 165 | dir.create("pipelines") 166 | writeLines( 167 | " 168 | #' @maestroFrequency 1 day 169 | #' @maestroRunIf stop('oh no') 170 | p1 <- function() { 171 | TRUE 172 | } 173 | ", 174 | con = "pipelines/conditionals.R" 175 | ) 176 | 177 | schedule <- build_schedule(quiet = TRUE) 178 | run_schedule( 179 | schedule, 180 | orch_frequency = "1 day", 181 | quiet = TRUE 182 | ) 183 | status <- get_status(schedule) 184 | }) 185 | 186 | expect_snapshot(status$success) 187 | expect_snapshot(status$invoked) 188 | expect_snapshot(last_run_errors()) 189 | }) 190 | 191 | test_that("Conditions that don't return a single boolean are handled", { 192 | withr::with_tempdir({ 193 | dir.create("pipelines") 194 | writeLines( 195 | " 196 | #' @maestroFrequency 1 day 197 | #' @maestroRunIf c(1, 2) 198 | p1 <- function() { 199 | TRUE 200 | } 201 | ", 202 | con = "pipelines/conditionals.R" 203 | ) 204 | 205 | schedule <- build_schedule(quiet = TRUE) 206 | run_schedule( 207 | schedule, 208 | orch_frequency = "1 day", 209 | quiet = TRUE 210 | ) 211 | status <- get_status(schedule) 212 | }) 213 | 214 | expect_snapshot(status$success) 215 | expect_snapshot(status$invoked) 216 | expect_snapshot(last_run_errors()) 217 | }) 218 | 219 | test_that("Empty maestroRunIf is ignored", { 220 | withr::with_tempdir({ 221 | dir.create("pipelines") 222 | writeLines( 223 | " 224 | #' @maestroFrequency 1 day 225 | #' @maestroRunIf 226 | p1 <- function() { 227 | TRUE 228 | } 229 | ", 230 | con = "pipelines/conditionals.R" 231 | ) 232 | 233 | schedule <- build_schedule(quiet = TRUE) 234 | run_schedule( 235 | schedule, 236 | orch_frequency = "1 day", 237 | quiet = FALSE 238 | ) 239 | status <- get_status(schedule) 240 | }) 241 | 242 | expect_snapshot(status$success) 243 | expect_snapshot(status$invoked) 244 | }) 245 | 246 | test_that("Branching pipelines execute with conditionals", { 247 | 248 | withr::with_tempdir({ 249 | dir.create("pipelines") 250 | writeLines( 251 | " 252 | #' @maestroFrequency 1 day 253 | a <- function() { 254 | 'b' 255 | } 256 | 257 | #' @maestroInputs a 258 | #' @maestroRunIf .input == 'b' 259 | b <- function(.input) { 260 | TRUE 261 | } 262 | 263 | #' @maestroInputs a 264 | #' @maestroRunIf .input == 'c' 265 | c <- function(.input) { 266 | TRUE 267 | } 268 | ", 269 | con = "pipelines/conditionals.R" 270 | ) 271 | 272 | schedule <- build_schedule(quiet = TRUE) 273 | run_schedule( 274 | schedule, 275 | orch_frequency = "1 day", 276 | quiet = FALSE 277 | ) 278 | status <- get_status(schedule) 279 | }) 280 | 281 | expect_snapshot(status$invoked) 282 | }) 283 | -------------------------------------------------------------------------------- /man/MaestroSchedule.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/MaestroSchedule.R 3 | \name{MaestroSchedule} 4 | \alias{MaestroSchedule} 5 | \title{Class for a schedule of pipelines} 6 | \description{ 7 | Class for a schedule of pipelines 8 | 9 | Class for a schedule of pipelines 10 | } 11 | \examples{ 12 | if (interactive()) { 13 | pipeline_dir <- tempdir() 14 | create_pipeline("my_new_pipeline", pipeline_dir, open = FALSE) 15 | schedule <- build_schedule(pipeline_dir = pipeline_dir) 16 | } 17 | } 18 | \section{Public fields}{ 19 | \if{html}{\out{
}} 20 | \describe{ 21 | \item{\code{PipelineList}}{object of type MaestroPipelineList} 22 | } 23 | \if{html}{\out{
}} 24 | } 25 | \section{Methods}{ 26 | \subsection{Public methods}{ 27 | \itemize{ 28 | \item \href{#method-MaestroSchedule-new}{\code{MaestroSchedule$new()}} 29 | \item \href{#method-MaestroSchedule-print}{\code{MaestroSchedule$print()}} 30 | \item \href{#method-MaestroSchedule-run}{\code{MaestroSchedule$run()}} 31 | \item \href{#method-MaestroSchedule-get_schedule}{\code{MaestroSchedule$get_schedule()}} 32 | \item \href{#method-MaestroSchedule-get_status}{\code{MaestroSchedule$get_status()}} 33 | \item \href{#method-MaestroSchedule-get_artifacts}{\code{MaestroSchedule$get_artifacts()}} 34 | \item \href{#method-MaestroSchedule-get_network}{\code{MaestroSchedule$get_network()}} 35 | \item \href{#method-MaestroSchedule-get_flags}{\code{MaestroSchedule$get_flags()}} 36 | \item \href{#method-MaestroSchedule-show_network}{\code{MaestroSchedule$show_network()}} 37 | \item \href{#method-MaestroSchedule-clone}{\code{MaestroSchedule$clone()}} 38 | } 39 | } 40 | \if{html}{\out{
}} 41 | \if{html}{\out{}} 42 | \if{latex}{\out{\hypertarget{method-MaestroSchedule-new}{}}} 43 | \subsection{Method \code{new()}}{ 44 | Create a MaestroSchedule object 45 | \subsection{Usage}{ 46 | \if{html}{\out{
}}\preformatted{MaestroSchedule$new(Pipelines = NULL)}\if{html}{\out{
}} 47 | } 48 | 49 | \subsection{Arguments}{ 50 | \if{html}{\out{
}} 51 | \describe{ 52 | \item{\code{Pipelines}}{list of MaestroPipelines} 53 | } 54 | \if{html}{\out{
}} 55 | } 56 | \subsection{Returns}{ 57 | MaestroSchedule 58 | } 59 | } 60 | \if{html}{\out{
}} 61 | \if{html}{\out{}} 62 | \if{latex}{\out{\hypertarget{method-MaestroSchedule-print}{}}} 63 | \subsection{Method \code{print()}}{ 64 | Print the schedule object 65 | \subsection{Usage}{ 66 | \if{html}{\out{
}}\preformatted{MaestroSchedule$print()}\if{html}{\out{
}} 67 | } 68 | 69 | \subsection{Returns}{ 70 | print 71 | } 72 | } 73 | \if{html}{\out{
}} 74 | \if{html}{\out{}} 75 | \if{latex}{\out{\hypertarget{method-MaestroSchedule-run}{}}} 76 | \subsection{Method \code{run()}}{ 77 | Run a MaestroSchedule 78 | \subsection{Usage}{ 79 | \if{html}{\out{
}}\preformatted{MaestroSchedule$run(..., quiet = FALSE, run_all = FALSE, n_show_next = 5)}\if{html}{\out{
}} 80 | } 81 | 82 | \subsection{Arguments}{ 83 | \if{html}{\out{
}} 84 | \describe{ 85 | \item{\code{...}}{arguments passed to MaestroPipelineList$run} 86 | 87 | \item{\code{quiet}}{whether or not to silence console messages} 88 | 89 | \item{\code{run_all}}{run all pipelines regardless of the schedule (default is \code{FALSE}) - useful for testing.} 90 | 91 | \item{\code{n_show_next}}{show the next n scheduled pipes} 92 | } 93 | \if{html}{\out{
}} 94 | } 95 | \subsection{Returns}{ 96 | invisible 97 | } 98 | } 99 | \if{html}{\out{
}} 100 | \if{html}{\out{}} 101 | \if{latex}{\out{\hypertarget{method-MaestroSchedule-get_schedule}{}}} 102 | \subsection{Method \code{get_schedule()}}{ 103 | Get the schedule as a data.frame 104 | \subsection{Usage}{ 105 | \if{html}{\out{
}}\preformatted{MaestroSchedule$get_schedule()}\if{html}{\out{
}} 106 | } 107 | 108 | \subsection{Returns}{ 109 | data.frame 110 | } 111 | } 112 | \if{html}{\out{
}} 113 | \if{html}{\out{}} 114 | \if{latex}{\out{\hypertarget{method-MaestroSchedule-get_status}{}}} 115 | \subsection{Method \code{get_status()}}{ 116 | Get status of the pipelines as a data.frame 117 | \subsection{Usage}{ 118 | \if{html}{\out{
}}\preformatted{MaestroSchedule$get_status()}\if{html}{\out{
}} 119 | } 120 | 121 | \subsection{Returns}{ 122 | data.frame 123 | } 124 | } 125 | \if{html}{\out{
}} 126 | \if{html}{\out{}} 127 | \if{latex}{\out{\hypertarget{method-MaestroSchedule-get_artifacts}{}}} 128 | \subsection{Method \code{get_artifacts()}}{ 129 | Get artifacts (return values) from the pipelines 130 | \subsection{Usage}{ 131 | \if{html}{\out{
}}\preformatted{MaestroSchedule$get_artifacts()}\if{html}{\out{
}} 132 | } 133 | 134 | \subsection{Returns}{ 135 | list 136 | } 137 | } 138 | \if{html}{\out{
}} 139 | \if{html}{\out{}} 140 | \if{latex}{\out{\hypertarget{method-MaestroSchedule-get_network}{}}} 141 | \subsection{Method \code{get_network()}}{ 142 | Get the network structure of the pipelines as an edge list (will be empty if there are no DAG pipelines) 143 | \subsection{Usage}{ 144 | \if{html}{\out{
}}\preformatted{MaestroSchedule$get_network()}\if{html}{\out{
}} 145 | } 146 | 147 | \subsection{Returns}{ 148 | data.frame 149 | } 150 | } 151 | \if{html}{\out{
}} 152 | \if{html}{\out{}} 153 | \if{latex}{\out{\hypertarget{method-MaestroSchedule-get_flags}{}}} 154 | \subsection{Method \code{get_flags()}}{ 155 | Get all pipeline flags as a long data.frame 156 | \subsection{Usage}{ 157 | \if{html}{\out{
}}\preformatted{MaestroSchedule$get_flags()}\if{html}{\out{
}} 158 | } 159 | 160 | \subsection{Returns}{ 161 | data.frame 162 | } 163 | } 164 | \if{html}{\out{
}} 165 | \if{html}{\out{}} 166 | \if{latex}{\out{\hypertarget{method-MaestroSchedule-show_network}{}}} 167 | \subsection{Method \code{show_network()}}{ 168 | Visualize the DAG relationships between pipelines in the schedule 169 | \subsection{Usage}{ 170 | \if{html}{\out{
}}\preformatted{MaestroSchedule$show_network()}\if{html}{\out{
}} 171 | } 172 | 173 | \subsection{Returns}{ 174 | interactive visualization 175 | } 176 | } 177 | \if{html}{\out{
}} 178 | \if{html}{\out{}} 179 | \if{latex}{\out{\hypertarget{method-MaestroSchedule-clone}{}}} 180 | \subsection{Method \code{clone()}}{ 181 | The objects of this class are cloneable with this method. 182 | \subsection{Usage}{ 183 | \if{html}{\out{
}}\preformatted{MaestroSchedule$clone(deep = FALSE)}\if{html}{\out{
}} 184 | } 185 | 186 | \subsection{Arguments}{ 187 | \if{html}{\out{
}} 188 | \describe{ 189 | \item{\code{deep}}{Whether to make a deep clone.} 190 | } 191 | \if{html}{\out{
}} 192 | } 193 | } 194 | } 195 | --------------------------------------------------------------------------------