├── .Rprofile ├── .gitignore ├── README.Rmd ├── README.md ├── README_files └── figure-gfm │ ├── kj-patch-1.png │ ├── kj_patch_kj-1.png │ ├── perf-error-line-1.png │ ├── perf_bt_charts-1.png │ ├── perf_build_times-1.png │ └── unnamed-chunk-1-1.png ├── _drake-kj.R ├── _drake-raschka.R ├── data └── fivek-simdat.pickle ├── duration-experiment ├── kuhn-johnson │ ├── nested-cv-h2o-kj.R │ ├── nested-cv-parsnip-kj.R │ ├── nested-cv-ranger-kj.R │ ├── nested-cv-sklearn-kj.R │ └── nested-cv-tune-kj.R ├── outputs │ ├── 0225-results.png │ ├── duration-pkg-tbl.png │ └── duration-runs.rds ├── package-sources-gt-tbl.R └── raschka │ ├── nested-cv-kj-raschka.R │ ├── nested-cv-mlr3-raschka.R │ ├── nested-cv-py-raschka.py │ └── nested-cv-retic-raschka.R ├── environment.yml ├── grids ├── elast-latin-params.pickle └── rf-latin-params.pickle ├── images └── ncv.png ├── nested-cross-validation-comparison.Rproj ├── palettes └── Forest Floor.ase ├── performance-experiment ├── Kuhn-Johnson │ ├── plan-kj.R │ └── r_make-kj.R ├── Raschka │ ├── plan-raschka.R │ └── r_make-raschka.R ├── functions │ ├── create-grids.R │ ├── create-models.R │ ├── create-ncv-objects.R │ ├── inner-tune.R │ ├── mlbench-data.R │ ├── ncv-compare.R │ ├── outer-cv.R │ └── run-ncv.R ├── output │ ├── perf-exp-output-r.csv │ └── perf-exp-output.csv └── packages.R ├── renv.lock └── renv ├── .gitignore ├── activate.R └── settings.dcf /.Rprofile: -------------------------------------------------------------------------------- 1 | source("renv/activate.R") 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/.gitignore -------------------------------------------------------------------------------- /README.Rmd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/README.Rmd -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/README.md -------------------------------------------------------------------------------- /README_files/figure-gfm/kj-patch-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/README_files/figure-gfm/kj-patch-1.png -------------------------------------------------------------------------------- /README_files/figure-gfm/kj_patch_kj-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/README_files/figure-gfm/kj_patch_kj-1.png -------------------------------------------------------------------------------- /README_files/figure-gfm/perf-error-line-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/README_files/figure-gfm/perf-error-line-1.png -------------------------------------------------------------------------------- /README_files/figure-gfm/perf_bt_charts-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/README_files/figure-gfm/perf_bt_charts-1.png -------------------------------------------------------------------------------- /README_files/figure-gfm/perf_build_times-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/README_files/figure-gfm/perf_build_times-1.png -------------------------------------------------------------------------------- /README_files/figure-gfm/unnamed-chunk-1-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/README_files/figure-gfm/unnamed-chunk-1-1.png -------------------------------------------------------------------------------- /_drake-kj.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/_drake-kj.R -------------------------------------------------------------------------------- /_drake-raschka.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/_drake-raschka.R -------------------------------------------------------------------------------- /data/fivek-simdat.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/data/fivek-simdat.pickle -------------------------------------------------------------------------------- /duration-experiment/kuhn-johnson/nested-cv-h2o-kj.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/kuhn-johnson/nested-cv-h2o-kj.R -------------------------------------------------------------------------------- /duration-experiment/kuhn-johnson/nested-cv-parsnip-kj.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/kuhn-johnson/nested-cv-parsnip-kj.R -------------------------------------------------------------------------------- /duration-experiment/kuhn-johnson/nested-cv-ranger-kj.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/kuhn-johnson/nested-cv-ranger-kj.R -------------------------------------------------------------------------------- /duration-experiment/kuhn-johnson/nested-cv-sklearn-kj.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/kuhn-johnson/nested-cv-sklearn-kj.R -------------------------------------------------------------------------------- /duration-experiment/kuhn-johnson/nested-cv-tune-kj.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/kuhn-johnson/nested-cv-tune-kj.R -------------------------------------------------------------------------------- /duration-experiment/outputs/0225-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/outputs/0225-results.png -------------------------------------------------------------------------------- /duration-experiment/outputs/duration-pkg-tbl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/outputs/duration-pkg-tbl.png -------------------------------------------------------------------------------- /duration-experiment/outputs/duration-runs.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/outputs/duration-runs.rds -------------------------------------------------------------------------------- /duration-experiment/package-sources-gt-tbl.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/package-sources-gt-tbl.R -------------------------------------------------------------------------------- /duration-experiment/raschka/nested-cv-kj-raschka.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/raschka/nested-cv-kj-raschka.R -------------------------------------------------------------------------------- /duration-experiment/raschka/nested-cv-mlr3-raschka.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/raschka/nested-cv-mlr3-raschka.R -------------------------------------------------------------------------------- /duration-experiment/raschka/nested-cv-py-raschka.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/raschka/nested-cv-py-raschka.py -------------------------------------------------------------------------------- /duration-experiment/raschka/nested-cv-retic-raschka.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/duration-experiment/raschka/nested-cv-retic-raschka.R -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/environment.yml -------------------------------------------------------------------------------- /grids/elast-latin-params.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/grids/elast-latin-params.pickle -------------------------------------------------------------------------------- /grids/rf-latin-params.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/grids/rf-latin-params.pickle -------------------------------------------------------------------------------- /images/ncv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/images/ncv.png -------------------------------------------------------------------------------- /nested-cross-validation-comparison.Rproj: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/nested-cross-validation-comparison.Rproj -------------------------------------------------------------------------------- /palettes/Forest Floor.ase: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/palettes/Forest Floor.ase -------------------------------------------------------------------------------- /performance-experiment/Kuhn-Johnson/plan-kj.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/Kuhn-Johnson/plan-kj.R -------------------------------------------------------------------------------- /performance-experiment/Kuhn-Johnson/r_make-kj.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/Kuhn-Johnson/r_make-kj.R -------------------------------------------------------------------------------- /performance-experiment/Raschka/plan-raschka.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/Raschka/plan-raschka.R -------------------------------------------------------------------------------- /performance-experiment/Raschka/r_make-raschka.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/Raschka/r_make-raschka.R -------------------------------------------------------------------------------- /performance-experiment/functions/create-grids.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/functions/create-grids.R -------------------------------------------------------------------------------- /performance-experiment/functions/create-models.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/functions/create-models.R -------------------------------------------------------------------------------- /performance-experiment/functions/create-ncv-objects.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/functions/create-ncv-objects.R -------------------------------------------------------------------------------- /performance-experiment/functions/inner-tune.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/functions/inner-tune.R -------------------------------------------------------------------------------- /performance-experiment/functions/mlbench-data.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/functions/mlbench-data.R -------------------------------------------------------------------------------- /performance-experiment/functions/ncv-compare.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/functions/ncv-compare.R -------------------------------------------------------------------------------- /performance-experiment/functions/outer-cv.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/functions/outer-cv.R -------------------------------------------------------------------------------- /performance-experiment/functions/run-ncv.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/functions/run-ncv.R -------------------------------------------------------------------------------- /performance-experiment/output/perf-exp-output-r.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/output/perf-exp-output-r.csv -------------------------------------------------------------------------------- /performance-experiment/output/perf-exp-output.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/output/perf-exp-output.csv -------------------------------------------------------------------------------- /performance-experiment/packages.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/performance-experiment/packages.R -------------------------------------------------------------------------------- /renv.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/renv.lock -------------------------------------------------------------------------------- /renv/.gitignore: -------------------------------------------------------------------------------- 1 | local/ 2 | lock/ 3 | library/ 4 | python/ 5 | staging/ 6 | -------------------------------------------------------------------------------- /renv/activate.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/renv/activate.R -------------------------------------------------------------------------------- /renv/settings.dcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ercbk/nested-cross-validation-comparison/HEAD/renv/settings.dcf --------------------------------------------------------------------------------