├── .Rbuildignore
├── .gitattributes
├── .github
    ├── .gitignore
    └── workflows
    │   ├── R-CMD-check.yaml
    │   ├── pkgdown.yaml
    │   ├── pr-commands.yaml
    │   └── test-coverage.yaml
├── .gitignore
├── DESCRIPTION
├── NAMESPACE
├── R
    └── modelDown.R
├── README.md
├── codecov.yml
├── inst
    └── extdata
    │   ├── modules
    │       ├── auditor
    │       │   ├── generator.R
    │       │   ├── styles.css
    │       │   └── template.html
    │       ├── drifter
    │       │   ├── generator.R
    │       │   ├── styles.css
    │       │   └── template.html
    │       ├── model_performance
    │       │   ├── generator.R
    │       │   ├── styles.css
    │       │   └── template.html
    │       ├── variable_importance
    │       │   ├── generator.R
    │       │   ├── styles.css
    │       │   └── template.html
    │       └── variable_response
    │       │   ├── generator.R
    │       │   ├── styles.css
    │       │   └── template.html
    │   └── template
    │       ├── base_template.html
    │       ├── index_template.html
    │       ├── link.svg
    │       ├── mi2template.css
    │       └── style.css
├── man
    ├── figures
    │   ├── auditor.png
    │   ├── importance.png
    │   ├── index.png
    │   ├── logo.png
    │   ├── modelDownSticker.png
    │   ├── performance.png
    │   └── response.png
    └── modelDown.Rd
├── modelDown.Rproj
├── paper.bib
├── paper.md
├── paper_images
    ├── model_performance.png
    └── variable_importance.png
├── pkgdown
    ├── _pkgdown.yml
    └── favicon
    │   ├── apple-touch-icon-120x120.png
    │   ├── apple-touch-icon-152x152.png
    │   ├── apple-touch-icon-180x180.png
    │   ├── apple-touch-icon-60x60.png
    │   ├── apple-touch-icon-76x76.png
    │   ├── apple-touch-icon.png
    │   ├── favicon-16x16.png
    │   ├── favicon-32x32.png
    │   └── favicon.ico
├── scripts
    └── regenerate_pkgdown_docs.R
└── tests
    ├── testthat.R
    └── testthat
        ├── test_modelDown.R
        └── test_parseExplainers.R


/.Rbuildignore:
--------------------------------------------------------------------------------
 1 | ^README.md$
 2 | ^.*\.Rproj$
 3 | ^\.Rproj\.user$
 4 | ^docs$
 5 | ^_pkgdown\.yml$
 6 | .travis.yml
 7 | vignettes/
 8 | ^paper.bib$
 9 | ^paper.md$
10 | ^cran-comments.md$
11 | paper_images/
12 | scripts/
13 | ^\.github$
14 | ^.*\.gif
15 | ^.*\.png
16 | ^pkgdown$
17 | ^codecov\.yml$
18 | 


--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | docs/* linguist-documentation
2 | man/* linguist-documentation
3 | misc/* linguist-documentation
4 | pkgdown/* linguist-documentation
5 | 


--------------------------------------------------------------------------------
/.github/.gitignore:
--------------------------------------------------------------------------------
1 | *.html
2 | 


--------------------------------------------------------------------------------
/.github/workflows/R-CMD-check.yaml:
--------------------------------------------------------------------------------
  1 | # NOTE: This workflow is overkill for most R packages
  2 | # check-standard.yaml is likely a better choice
  3 | # usethis::use_github_action("check-standard") will install it.
  4 | #
  5 | # For help debugging build failures open an issue on the RStudio community with the 'github-actions' tag.
  6 | # https://community.rstudio.com/new-topic?category=Package%20development&tags=github-actions
  7 | on:
  8 |   push:
  9 |     branches:
 10 |       - master
 11 |   pull_request:
 12 |     branches:
 13 |       - master
 14 | 
 15 | name: R-CMD-check
 16 | 
 17 | jobs:
 18 |   R-CMD-check:
 19 |     runs-on: ${{ matrix.config.os }}
 20 | 
 21 |     name: ${{ matrix.config.os }} (${{ matrix.config.r }})
 22 | 
 23 |     strategy:
 24 |       fail-fast: false
 25 |       matrix:
 26 |         config:
 27 |           - {os: macOS-latest,   r: 'release'}
 28 |           - {os: windows-latest, r: 'release'}
 29 |           - {os: windows-latest, r: '3.6'}
 30 |           - {os: ubuntu-16.04,   r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest", http-user-agent: "R/4.0.0 (ubuntu-16.04) R (4.0.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions" }
 31 |           - {os: ubuntu-16.04,   r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
 32 |           - {os: ubuntu-16.04,   r: 'oldrel',  rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
 33 |           - {os: ubuntu-16.04,   r: '3.5',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
 34 | 
 35 |     env:
 36 |       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
 37 |       RSPM: ${{ matrix.config.rspm }}
 38 |       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
 39 | 
 40 |     steps:
 41 |       - uses: actions/checkout@v2
 42 | 
 43 |       - uses: r-lib/actions/setup-r@master
 44 |         with:
 45 |           r-version: ${{ matrix.config.r }}
 46 |           http-user-agent: ${{ matrix.config.http-user-agent }}
 47 | 
 48 |       - uses: r-lib/actions/setup-pandoc@master
 49 | 
 50 |       - name: Query dependencies
 51 |         run: |
 52 |           install.packages('remotes')
 53 |           saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
 54 |           writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
 55 |         shell: Rscript {0}
 56 | 
 57 |       - name: Cache R packages
 58 |         if: runner.os != 'Windows'
 59 |         uses: actions/cache@v2
 60 |         with:
 61 |           path: ${{ env.R_LIBS_USER }}
 62 |           key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
 63 |           restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
 64 | 
 65 |       - name: Install system dependencies
 66 |         if: runner.os == 'Linux'
 67 |         run: |
 68 |           while read -r cmd
 69 |           do
 70 |             eval sudo $cmd
 71 |           done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "16.04"))')
 72 | 
 73 |       - name: Install dependencies
 74 |         run: |
 75 |           remotes::install_deps(dependencies = TRUE)
 76 |           remotes::install_cran("rcmdcheck")
 77 |         shell: Rscript {0}
 78 | 
 79 |       - name: Session info
 80 |         run: |
 81 |           options(width = 100)
 82 |           pkgs <- installed.packages()[, "Package"]
 83 |           sessioninfo::session_info(pkgs, include_base = TRUE)
 84 |         shell: Rscript {0}
 85 | 
 86 |       - name: Check
 87 |         env:
 88 |           _R_CHECK_CRAN_INCOMING_: false
 89 |         run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran", "--run-donttest", "--run-dontrun"), error_on = "warning", check_dir = "check")
 90 |         shell: Rscript {0}
 91 | 
 92 |       - name: Show testthat output
 93 |         if: always()
 94 |         run: find check -name 'testthat.Rout*' -exec cat '{}' \; || true
 95 |         shell: bash
 96 | 
 97 |       - name: Upload check results
 98 |         if: failure()
 99 |         uses: actions/upload-artifact@main
100 |         with:
101 |           name: ${{ runner.os }}-r${{ matrix.config.r }}-results
102 |           path: check
103 | 


--------------------------------------------------------------------------------
/.github/workflows/pkgdown.yaml:
--------------------------------------------------------------------------------
 1 | on:
 2 |   push:
 3 |     branches: master
 4 | 
 5 | name: pkgdown
 6 | 
 7 | jobs:
 8 |   pkgdown:
 9 |     runs-on: macOS-latest
10 |     env:
11 |       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
12 |     steps:
13 |       - uses: actions/checkout@v2
14 | 
15 |       - uses: r-lib/actions/setup-r@master
16 | 
17 |       - uses: r-lib/actions/setup-pandoc@master
18 | 
19 |       - name: Query dependencies
20 |         run: |
21 |           install.packages('remotes')
22 |           saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
23 |           writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
24 |         shell: Rscript {0}
25 | 
26 |       - name: Cache R packages
27 |         uses: actions/cache@v2
28 |         with:
29 |           path: ${{ env.R_LIBS_USER }}
30 |           key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
31 |           restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
32 | 
33 |       - name: Install dependencies
34 |         run: |
35 |           remotes::install_deps(dependencies = TRUE)
36 |           install.packages("pkgdown")
37 |           remotes::install_github("ModelOriented/DrWhyTemplate")
38 |         shell: Rscript {0}
39 | 
40 |       - name: Install package
41 |         run: R CMD INSTALL .
42 | 
43 |       - name: Deploy package
44 |         run: |
45 |           git config --local user.email "actions@github.com"
46 |           git config --local user.name "GitHub Actions"
47 |           Rscript -e 'pkgdown::deploy_to_branch(new_process = FALSE)'
48 | 


--------------------------------------------------------------------------------
/.github/workflows/pr-commands.yaml:
--------------------------------------------------------------------------------
 1 | on:
 2 |   issue_comment:
 3 |     types: [created]
 4 | name: Commands
 5 | jobs:
 6 |   document:
 7 |     if: startsWith(github.event.comment.body, '/document')
 8 |     name: document
 9 |     runs-on: macOS-latest
10 |     env:
11 |       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
12 |     steps:
13 |       - uses: actions/checkout@v2
14 |       - uses: r-lib/actions/pr-fetch@master
15 |         with:
16 |           repo-token: ${{ secrets.GITHUB_TOKEN }}
17 |       - uses: r-lib/actions/setup-r@master
18 |       - name: Install dependencies
19 |         run: Rscript -e 'install.packages(c("remotes", "roxygen2"))' -e 'remotes::install_deps(dependencies = TRUE)'
20 |       - name: Document
21 |         run: Rscript -e 'roxygen2::roxygenise()'
22 |       - name: commit
23 |         run: |
24 |           git config --local user.email "actions@github.com"
25 |           git config --local user.name "GitHub Actions"
26 |           git add man/\* NAMESPACE
27 |           git commit -m 'Document'
28 |       - uses: r-lib/actions/pr-push@master
29 |         with:
30 |           repo-token: ${{ secrets.GITHUB_TOKEN }}
31 |   style:
32 |     if: startsWith(github.event.comment.body, '/style')
33 |     name: style
34 |     runs-on: macOS-latest
35 |     env:
36 |       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
37 |     steps:
38 |       - uses: actions/checkout@v2
39 |       - uses: r-lib/actions/pr-fetch@master
40 |         with:
41 |           repo-token: ${{ secrets.GITHUB_TOKEN }}
42 |       - uses: r-lib/actions/setup-r@master
43 |       - name: Install dependencies
44 |         run: Rscript -e 'install.packages("styler")'
45 |       - name: Style
46 |         run: Rscript -e 'styler::style_pkg()'
47 |       - name: commit
48 |         run: |
49 |           git config --local user.email "actions@github.com"
50 |           git config --local user.name "GitHub Actions"
51 |           git add \*.R
52 |           git commit -m 'Style'
53 |       - uses: r-lib/actions/pr-push@master
54 |         with:
55 |           repo-token: ${{ secrets.GITHUB_TOKEN }}
56 | 


--------------------------------------------------------------------------------
/.github/workflows/test-coverage.yaml:
--------------------------------------------------------------------------------
 1 | on:
 2 |   push:
 3 |     branches:
 4 |       - master
 5 |   pull_request:
 6 |     branches:
 7 |       - master
 8 | 
 9 | name: test-coverage
10 | 
11 | jobs:
12 |   test-coverage:
13 |     runs-on: macOS-latest
14 |     env:
15 |       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
16 |     steps:
17 |       - uses: actions/checkout@v2
18 | 
19 |       - uses: r-lib/actions/setup-r@master
20 | 
21 |       - uses: r-lib/actions/setup-pandoc@master
22 | 
23 |       - name: Query dependencies
24 |         run: |
25 |           install.packages('remotes')
26 |           saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
27 |           writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
28 |         shell: Rscript {0}
29 | 
30 |       - name: Cache R packages
31 |         uses: actions/cache@v2
32 |         with:
33 |           path: ${{ env.R_LIBS_USER }}
34 |           key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
35 |           restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
36 | 
37 |       - name: Install dependencies
38 |         run: |
39 |           install.packages(c("remotes"))
40 |           remotes::install_deps(dependencies = TRUE)
41 |           remotes::install_cran("covr")
42 |         shell: Rscript {0}
43 | 
44 |       - name: Test coverage
45 |         run: covr::codecov()
46 |         shell: Rscript {0}
47 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # History files
 2 | .Rhistory
 3 | .Rapp.history
 4 | 
 5 | # Session Data files
 6 | .RData
 7 | 
 8 | # Example code in package build process
 9 | *-Ex.R
10 | 
11 | # Output files from R CMD build
12 | /*.tar.gz
13 | 
14 | # Output files from R CMD check
15 | /*.Rcheck/
16 | 
17 | # RStudio files
18 | .Rproj.user/
19 | 
20 | # produced vignettes
21 | vignettes/*.html
22 | vignettes/*.pdf
23 | 
24 | # OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3
25 | .httr-oauth
26 | 
27 | # knitr and R markdown default cache directories
28 | /*_cache/
29 | /cache/
30 | 
31 | # Temporary files created by R markdown
32 | *.utf8.md
33 | *.knit.md
34 | 
35 | # Shiny token, see https://shiny.rstudio.com/articles/shinyapps.html
36 | rsconnect/
37 | .Rproj.user
38 | #modelDown output
39 | output/
40 | 
41 | # JOSS review compilation files
42 | apa.csl
43 | latex.template
44 | compile
45 | paper.pdf
46 | joss-logo.png
47 | 


--------------------------------------------------------------------------------
/DESCRIPTION:
--------------------------------------------------------------------------------
 1 | Package: modelDown
 2 | Title: Make Static HTML Website for Predictive Models
 3 | Version: 1.1
 4 | Authors@R: c(person("Przemyslaw", "Biecek", email = "przemyslaw.biecek@gmail.com", role = c("aut")), 
 5 |     person("Magda", "Tatarynowicz", email = "magda.tatarynowicz@gmail.com", role = c("aut")), 
 6 |     person("Kamil", "Romaszko", email = "kamil.romaszko@gmail.com", role = c("aut", "cre")), 
 7 |     person("Mateusz", "Urbanski", email = "matiszak@gmail.com", role = c("aut")))
 8 | Description: Website generator with HTML summaries for predictive models.
 9 |     This package uses 'DALEX' explainers to describe global model behavior. 
10 |     We can see how well models behave (tabs: Model Performance, Auditor),
11 |     how much each variable contributes to predictions (tabs: Variable Response) 
12 |     and which variables are the most important for a given model (tabs: Variable Importance).
13 |     We can also compare Concept Drift for pairs of models (tabs: Drifter).
14 |     Additionally, data available on the website can be easily recreated in current R session.
15 |     Work on this package was financially supported by the NCN Opus grant 2017/27/B/ST6/01307 
16 |     at Warsaw University of Technology, Faculty of Mathematics and Information Science.
17 | Depends: R (>= 3.4.0)
18 | License: Apache License 2.0
19 | Encoding: UTF-8
20 | LazyData: true
21 | Imports: DALEX (>= 1.0),
22 |     auditor (>= 0.3.0),
23 |     ggplot2 (>= 3.1.0),
24 |     whisker (>= 0.3-2),
25 |     DT (>= 0.4),
26 |     kableExtra (>= 0.9.0),
27 |     psych (>= 1.8.4),
28 |     archivist (>= 2.1.0),
29 |     svglite (>= 1.2.1),
30 |     devtools (>= 2.0.1),
31 |     breakDown (>= 0.1.6),
32 |     drifter (>= 0.2.1)
33 | Suggests: 
34 |     ranger,
35 |     testthat,
36 |     useful,
37 |     covr
38 | RoxygenNote: 7.1.1
39 | URL: https://github.com/ModelOriented/modelDown
40 | BugReports: https://github.com/ModelOriented/modelDown/issues
41 | 


--------------------------------------------------------------------------------
/NAMESPACE:
--------------------------------------------------------------------------------
 1 | # Generated by roxygen2: do not edit by hand
 2 | 
 3 | export(modelDown)
 4 | import(ggplot2)
 5 | import(kableExtra)
 6 | import(whisker)
 7 | importFrom(grDevices,svg)
 8 | importFrom(graphics,plot)
 9 | importFrom(utils,capture.output)
10 | importFrom(utils,tail)
11 | 


--------------------------------------------------------------------------------
/R/modelDown.R:
--------------------------------------------------------------------------------
  1 | #' Generates a website with HTML summaries for predictive models
  2 | #'
  3 | #' @param ... one or more explainers created with \code{DALEX::explain()} function. Pair of explainer could be provided to check drift of models
  4 | #' @param modules modules that should be included in the website
  5 | #' @param output_folder folder where the website will be saved
  6 | #' @param repository_name name of local archivist repository that will be created
  7 | #' @param should_open_website should generated website be automatically opened in default browser
  8 | #'
  9 | #' @details
 10 | #' Additional arguments that could by passed by name:
 11 | #' \itemize{
 12 | #'   \item{remote_repository_path} {Path to remote repository that stores folder with archivist repository. If not provided, links to local repository will be shown.}
 13 | #'   \item{device} {Device to use. Tested for "png" and "svg", but values from \code{ggplot2::ggsave} function should be working fine. Defaults to "png".}
 14 | #'   \item{vr.vars} {variables which will be examined in Variable Response module. Defaults to all variables. Example vr.vars = c("var1", "var2")}
 15 | #'   \item{vr.type} {types of examinations which will be conducteed in Variable Response module. Defaults to "partial". Example vr.type = c("partial", "conditional", "accumulated")}
 16 | #' }
 17 | #'
 18 | #' @export
 19 | #' @import kableExtra
 20 | #' @import whisker
 21 | #' @import ggplot2
 22 | #' @importFrom grDevices svg
 23 | #' @importFrom graphics plot
 24 | #' @importFrom utils capture.output tail
 25 | #' @author Przemysław Biecek, Magda Tatarynowicz, Kamil Romaszko, Mateusz Urbański
 26 | #' @examples
 27 | #' if(FALSE){
 28 | #' require("ranger")
 29 | #' require("breakDown")
 30 | #' require("DALEX")
 31 | #'
 32 | #'
 33 | #' # Generate simple modelDown page
 34 | #' HR_data_selected <- HR_data[1000:3000,]
 35 | #' HR_glm_model <- glm(left~., HR_data_selected, family = "binomial")
 36 | #' explainer_glm <- explain(HR_glm_model, data=HR_data_selected, y = HR_data_selected$left)
 37 | #'
 38 | #' modelDown::modelDown(explainer_glm,
 39 | #'                      modules = c("model_performance", "variable_importance",
 40 | #'                                  "variable_response"),
 41 | #'                      output_folder = tempdir(),
 42 | #'                      repository_name = "HR",
 43 | #'                      device = "png",
 44 | #'                      vr.vars= c("average_montly_hours"),
 45 | #'                      vr.type = "partial")
 46 | #'
 47 | #' # More complex example with all modules
 48 | #' HR_ranger_model <- ranger(as.factor(left) ~ .,
 49 | #'                       data = HR_data, num.trees = 500, classification = TRUE, probability = TRUE)
 50 | #' explainer_ranger <- explain(HR_ranger_model,
 51 | #'                       data = HR_data, y = HR_data$left, function(model, data) {
 52 | #'  return(predict(model, data)$prediction[,2])
 53 | #' }, na.rm=TRUE)
 54 | #'
 55 | #' # Two glm models used for drift detection
 56 | #' HR_data1 <- HR_data[1:4000,]
 57 | #' HR_data2 <- HR_data[4000:nrow(HR_data),]
 58 | #' HR_glm_model1 <- glm(left~., HR_data1, family = "binomial")
 59 | #' HR_glm_model2 <- glm(left~., HR_data2, family = "binomial")
 60 | #' explainer_glm1 <- explain(HR_glm_model1, data=HR_data1, y = HR_data1$left)
 61 | #' explainer_glm2 <- explain(HR_glm_model2, data=HR_data2, y = HR_data2$left)
 62 | #'
 63 | #' modelDown::modelDown(list(explainer_glm1, explainer_glm2),
 64 | #'   modules = c("auditor", "drifter", "model_performance", "variable_importance",
 65 | #'               "variable_response"),
 66 | #'   output_folder = tempdir(),
 67 | #'   repository_name = "HR",
 68 | #'   remote_repository_path = "some_user/remote_repo_name",
 69 | #'   device = "png",
 70 | #'   vr.vars= c("average_montly_hours", "time_spend_company"),
 71 | #'   vr.type = "partial")
 72 | #' }
 73 | modelDown <- function(...,
 74 |                       modules = c("auditor", "drifter", "model_performance", "variable_importance", "variable_response"),
 75 |                       output_folder="output",
 76 |                       repository_name="repository",
 77 |                       should_open_website=interactive()) {
 78 | 
 79 |   args <- list(..., version=1.0 )
 80 |   #named arguments are options (except those specified after ... in function definition)
 81 |   options <- args[names(args) != ""]
 82 |   options[["output_folder"]] <- output_folder
 83 |   options[["repository_name"]] <- repository_name
 84 | 
 85 |   #unnamed arguments are explainers
 86 |   explainers_list <- args[names(args) == ""]
 87 | 
 88 |   explainers_parsed <- parseExplainers(explainers_list)
 89 |   explainers <- explainers_parsed$basic_explainers
 90 |   drifter_explainer_pairs <- explainers_parsed$drifter_explainer_pairs
 91 | 
 92 |   validateParameters(explainers, options, modules, should_open_website)
 93 | 
 94 |   # Do not render drifter tab if there are no explainer pairs
 95 |   if(length(drifter_explainer_pairs) == 0) {
 96 |     modules <- modules['drifter' != modules]
 97 |   }
 98 | 
 99 |   ensureOutputFolderStructureExist(output_folder);
100 |   do.call(file.remove, list(list.files(output_folder, full.names = TRUE, recursive = TRUE)))
101 | 
102 |   # create local repository
103 |   repository <- file.path(output_folder, options[["repository_name"]])
104 |   archivist::createLocalRepo(repoDir = repository)
105 | 
106 |   # save explainers
107 |   for(explainer in explainers_list){
108 |     if(class(explainer) == "list"){
109 |       saveRDS(explainer[[1]], file = paste0(output_folder,"/explainers/", explainer[[1]]$label, "_new", ".rda"))
110 |       saveRDS(explainer[[2]], file = paste0(output_folder,"/explainers/", explainer[[2]]$label, "_old", ".rda"))
111 |     } else {
112 |       saveRDS(explainer, file = paste0(output_folder,"/explainers/", explainer$label, ".rda"))
113 |     }
114 |   }
115 | 
116 |   #save session info
117 |   session_info <- devtools::session_info()
118 |   writeLines(capture.output(session_info), paste0(output_folder,"/session_info/session_info.txt"))
119 |   save(session_info, file = paste0(output_folder,"/session_info/session_info.rda"))
120 | 
121 |   copyAssets(system.file("extdata", "template", package = "modelDown"), output_folder)
122 | 
123 |   generated_modules <- generateModules(modules, output_folder, explainers, drifter_explainer_pairs, options)
124 | 
125 |   renderModules(generated_modules, output_folder)
126 |   renderMainPage(generated_modules, output_folder, explainers, explainers_list, options)
127 |   if(should_open_website){
128 |     utils::browseURL(file.path(output_folder, "index.html"))
129 |   }
130 | }
131 | 
132 | .onLoad <- function(libname, pkgname) {
133 |   op <- options()
134 |   op.modelDown <- list(
135 |     modelDown.default_font_size = 16,
136 |     modelDown.default_device = "png"
137 |   )
138 |   toset <- !(names(op.modelDown) %in% names(op))
139 |   if(any(toset)) options(op.modelDown[toset])
140 | 
141 |   invisible()
142 | }
143 | 
144 | validateParameters <- function(explainers, options, modules, should_open_website){
145 |   validateExplainers(explainers)
146 |   validateOptions(options, explainers)
147 | 
148 |   if(!is.logical((should_open_website))){
149 |     stop("Parameter 'should_open_website' has to be logical")
150 |   }
151 | 
152 |   # todo - put modules names into one variable
153 |   correct_modules <- c("auditor", "drifter", "model_performance", "variable_importance", "variable_response")
154 |   if(!all(modules %in% correct_modules)){
155 |     stop("Parameter 'modules' contains invalid names. Please refer to documentation.")
156 |   }
157 | }
158 | 
159 | validateExplainers <- function(explainers){
160 |   if(length(explainers) == 0){
161 |     stop("At least one explainer must be provided")
162 |   }
163 | 
164 |   for(explainer in explainers){
165 |     if(class(explainer) != "explainer"){
166 |       stop("All explainers must be of class 'explainer' (generated by explain() from DALEX package)")
167 |     }
168 | 
169 |     if(any(sapply(explainer$data, class) == "character")){
170 |       print("Found character variables:")
171 |       print(names(explainer$data)[sapply(explainer$data, class) == "character"])
172 |       stop("Character variables are not supported! Please remove them or convert to factors")
173 |     }
174 |   }
175 | 
176 |   # Check if explainers have the same columns
177 |   # Comparing with first explainer
178 |   explainer1 <- explainers[[1]]
179 | 
180 |   for(explainer2 in tail(explainers,-1)){
181 |     names_1 <- names(explainer1$data)
182 |     names_2 <- names(explainer2$data)
183 |     if(length(c(setdiff(names_1, names_2), setdiff(names_2, names_1))) > 0) {
184 |       stop("Explainers data variables must be identical")
185 |     }
186 |   }
187 | }
188 | 
189 | validateOptions <- function(options, explainers){
190 |   vr.type <- options[["vr.type"]]
191 |   if(!is.null(vr.type) && !vr.type %in% c("partial", "conditional", "accumulated")){
192 |     stop("Invalid 'vr.type' value. Must be 'partial', 'conditional' or 'accumulated'.")
193 |   }
194 | 
195 |   vr.vars <- options[["vr.vars"]]
196 |   if(!is.null(vr.vars)){
197 |     explainer <- explainers[[1]]
198 |     if(!all(vr.vars %in% colnames(explainer$data))) {
199 |       stop("All variables in 'vr.vars' must be contained in data frame columns.")
200 |     }
201 |   }
202 | 
203 |   device <- options[["device"]]
204 |   if(!is.null(device)){
205 |     suggested_devices <- c("png", "svg")
206 |     # available_devices from ggplot2::ggsave() documentation
207 |     available_devices <- c("eps", "ps", "tex", "pdf", "jpeg", "tiff", "png", "bmp", "svg", "wmf")
208 |     if(!device %in% available_devices){
209 |       stop('Device parameter is incorrect. Please provide one of "eps", "ps", "tex", "pdf", "jpeg", "tiff", "png", "bmp", "svg", "wmf"')
210 |     }
211 |     if(!device %in% suggested_devices){
212 |       warning('Package not tested for specified device. Displaying and saving plot images may not work as expected.')
213 |       warning("It is recommended to use 'png' or 'svg' as device.")
214 |     }
215 |   }
216 | }
217 | 
218 | parseExplainers <- function(explainers) {
219 |   basic_explainers <- list()
220 |   drifter_explainer_pairs <- list()
221 | 
222 |   basic_i <- 1
223 |   drifter_i <- 1
224 |   for(explainer in explainers) {
225 |     if(!(class(explainer) == "explainer" || length(explainer) <= 2)) {
226 |       stop("Multiple explainers should be passed in list and length shouldn't be higher than 2.\nUse:\nmodelDown(list(explainer_old, explainer_new), ...)")
227 |     }
228 | 
229 |     if(length(explainer) == 2) {
230 |       drifter_explainer_pairs[[drifter_i]] <- explainer
231 |       basic_explainers[[basic_i]]  <- explainer[[1]]
232 |       drifter_i <- drifter_i + 1
233 |     } else{
234 |       basic_explainers[[basic_i]] <- explainer
235 |     }
236 |     basic_i <- basic_i + 1
237 |   }
238 | 
239 |   result <- list(basic_explainers=basic_explainers, drifter_explainer_pairs=drifter_explainer_pairs)
240 |   return(result)
241 | }
242 | 
243 | getPlotSettings <- function(options, options_prefix = NULL, default_font_size = getOption("modelDown.default_font_size"), default_device = getOption("modelDown.default_device")) {
244 | 
245 |   if(!is.null(options_prefix)) {
246 |     font_size_variable <- paste(options_prefix, ".font_size", sep = "")
247 |     device_variable <- paste(options_prefix, ".device", sep = "")
248 |   }
249 | 
250 |   font_size <- getVarOrDefault(options, font_size_variable, "font_size", default_value = default_font_size)
251 |   device <- getVarOrDefault(options, device_variable, "device", default_value = default_device)
252 | 
253 |   return(list(
254 |     font_size = font_size,
255 |     device = device
256 |   ))
257 | }
258 | 
259 | getVarOrDefault <- function(options, var1, var2, default_value) {
260 |   if(!is.null(var1)) {
261 |     value <- options[[var1]]
262 |   }
263 |   if(is.null(value)) {
264 |     value <- options[[var2]]
265 |   }
266 |   if(is.null(value)) {
267 |     value <- default_value
268 |   }
269 |   return(value)
270 | }
271 | 
272 | save_to_repository <- function(artifact, options){
273 |   repository <- file.path(options[["output_folder"]], options[["repository_name"]])
274 |   hash <- archivist::saveToLocalRepo(artifact, repoDir=repository)
275 | 
276 |   remote_path <- options[["remote_repository_path"]]
277 |   link <- ''
278 |   if(is.null(remote_path)) {
279 |     link <- paste('archivist::loadFromLocalRepo(md5hash = "', hash, '", ', 'repoDir = "', repository,'")', sep = '')
280 |   }
281 |   else {
282 |     link <- paste('archivist::aread("', remote_path, '/', options[["repository_name"]], '/', hash, '")', sep = '')
283 |   }
284 |   return(link)
285 | }
286 | 
287 | makeGeneratorEnvironment <- function() {
288 |   e <- new.env()
289 |   e$getPlotSettings <- getPlotSettings
290 |   e$save_to_repository <- save_to_repository
291 |   return(e)
292 | }
293 | 
294 | copyAssets <- function(from, to) {
295 |   files <- list.files(from)
296 |   asset_files <- files[grepl(".*.(css|svg|png|gif)", files)]
297 |   asset_files_paths <- unlist(lapply(asset_files, function(name) {file.path(from, name)}))
298 |   file.copy(asset_files_paths, to, recursive=TRUE, overwrite = TRUE)
299 |   return(asset_files)
300 | }
301 | 
302 | generateModules <- function(modules_names, output_folder, explainers, drifter_explainer_pairs, options) {
303 | 
304 |   result <- lapply(modules_names, function(module_name) {
305 |     tryCatch(
306 |       generateModule(module_name, output_folder, explainers, drifter_explainer_pairs, options)
307 |       , error = function(err) {
308 |         warning(paste(
309 |           "Module '", module_name, "' generation failed. Skipping it.",
310 |           "The detailed error is: ", err
311 |           , sep = ""))
312 |       }
313 |     )
314 |   })
315 | 
316 |   result <- result[lapply(result, is.list) == TRUE]
317 | 
318 |   return(result)
319 | }
320 | 
321 | generateModule <- function(module_name, output_folder, explainers, drifter_explainer_pairs, options) {
322 |     print(paste("Generating ", module_name, "...", sep = ""))
323 |     generator_path <-
324 |       system.file("extdata", "modules", module_name, "generator.R", package = "modelDown")
325 |     generator_env <- makeGeneratorEnvironment()
326 |     source(generator_path, local = generator_env)
327 | 
328 |     module_folder <- file.path(output_folder, module_name)
329 |     img_folder <- file.path(module_folder, "img")
330 |     createDirIfNotExists(module_folder)
331 |     createDirIfNotExists(img_folder)
332 |     if(module_name == "drifter") {
333 |       data <- generator_env$generator(drifter_explainer_pairs, options, img_folder)
334 |     } else {
335 |       data <- generator_env$generator(explainers, options, img_folder)
336 |     }
337 |     return(data)
338 | }
339 | 
340 | ensureOutputFolderStructureExist <- function(output_folder) {
341 |   createDirIfNotExists(output_folder)
342 | 
343 |   img_folder_path <- file.path(output_folder, "explainers")
344 |   createDirIfNotExists(img_folder_path)
345 | 
346 |   img_folder_path <- file.path(output_folder, "img")
347 |   createDirIfNotExists(img_folder_path)
348 | 
349 |   session_folder_path <- file.path(output_folder, "session_info")
350 |   createDirIfNotExists(session_folder_path)
351 | }
352 | 
353 | renderPage <- function(content, modules, output_path, root_path, extra_css = c()) {
354 | 
355 |   menu_links <- lapply(modules, function(module) {
356 |     return(list(
357 |       name=module[['display_name']],
358 |       link=paste(module[['name']], '/index.html', sep="")
359 |     ))
360 |   })
361 | 
362 |   data <- list(
363 |     content = content,
364 |     menu_links = menu_links,
365 |     datetime = Sys.time(),
366 |     root_path = root_path,
367 |     extra_css = extra_css
368 |   )
369 | 
370 |   iteratelist(data[['menu_links']], name='menu_links')
371 | 
372 |   base_template_path <-
373 |     system.file("extdata", "template", "base_template.html", package = "modelDown")
374 |   base_template <- readLines(base_template_path)
375 |   page <- whisker.render(base_template, data)
376 |   file.create(output_path)
377 |   writeLines(page, output_path)
378 | }
379 | 
380 | renderModules <- function(modules, output_folder) {
381 |   lapply(modules, function(module) {
382 |     module_path <- file.path("extdata", "modules", module[['name']])
383 |     content_template <-
384 |       readLines(system.file(module_path, "template.html", package = "modelDown"))
385 | 
386 |     output_module_folder <- file.path(output_folder, module[['name']])
387 |     createDirIfNotExists(output_module_folder)
388 | 
389 |     copied_assets <- copyAssets(system.file(module_path, package = "modelDown"), output_module_folder)
390 |     content <- whisker.render(content_template, module[['data']])
391 |     output_path <- file.path(output_module_folder, "index.html")
392 |     renderPage(content, modules, output_path, "../", copied_assets)
393 |   })
394 | }
395 | 
396 | renderMainPage <- function(modules, output_folder, explainers, explainers_list, options) {
397 |   data_set <- explainers[[1]]$data
398 |   numeric_columns <- which(sapply(data_set, class) != "factor")
399 |   factor_columns <- which(sapply(data_set, class) == "factor")
400 |   variables_data <- kable_styling(kable(psych::describe(data_set[,numeric_columns])), bootstrap_options = c("responsive", "bordered", "hover"))
401 | 
402 |   main_page_data <- list(
403 |     explainers = renderExplainersList(explainers_list),
404 |     data_summary = variables_data,
405 |     factor_summary = renderFactorTables(data_set, factor_columns),
406 |     observations_count = nrow(explainers[[1]]$data),
407 |     columns_count = ncol(explainers[[1]]$data)
408 |   )
409 | 
410 |   content_template <-
411 |     readLines(system.file("extdata", "template", "index_template.html", package = "modelDown"))
412 |   content <- whisker.render(content_template, main_page_data)
413 |   output_path <- file.path(output_folder, "index.html")
414 |   renderPage(content, modules, output_path, "./")
415 | }
416 | download_link <- function(label, download_path){
417 |   link_element = paste0(
418 |     "<li>",
419 |    label,
420 |     " <a href='explainers/",
421 |    download_path,
422 |     ".rda'>(download)</a></li>"
423 |   )
424 |   return(link_element)
425 | }
426 | renderExplainersList <- function(explainers){
427 |   explainers_ul <- "<ul>"
428 |   for(explainer in explainers){
429 |     if(class(explainer)=="list"){
430 |       explainers_ul <-
431 |         paste(
432 |           explainers_ul,
433 |           "<li>",
434 |           explainer[[1]]$label,
435 |           "<ul>",
436 |           download_link("new", paste0(explainer[[1]]$label, "_new")),
437 |           download_link("old", paste0(explainer[[2]]$label, "_old")),
438 |           "</ul>",
439 |           "</li>",
440 |           sep = ""
441 |         )
442 |     }else{
443 |       explainers_ul <-
444 |         paste(
445 |           explainers_ul, download_link(explainer$label, explainer$label),
446 |           sep = ""
447 |         )
448 |     }
449 |   }
450 |   explainers_ul <- paste(explainers_ul, "</ul>", sep = "")
451 |   explainers_ul
452 | }
453 | 
454 | renderFactorTables <- function(data_set, factor_columns){
455 |   factor_data <- vector()
456 |   factor_data <- NULL
457 | 
458 |   if(length(factor_columns) > 0) {
459 |     for(i in 1:length(factor_columns)){
460 |       column_number <- factor_columns[[i]]
461 |       column_name <- names(factor_columns)[i]
462 |       temp_table <- kable_styling(kable(table(data_set[, column_number]), col.names = c(column_name, "Frequency")),bootstrap_options = c("responsive", "bordered", "hover"),full_width = FALSE)
463 |       factor_data <- paste(factor_data, temp_table, sep="<br>")
464 |     }
465 |   }
466 | 
467 |   return(factor_data)
468 | }
469 | 
470 | createDirIfNotExists <- function(path) {
471 |   if(!dir.exists(path)) {
472 |     dir.create(path)
473 |   }
474 | }
475 | 
476 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # modelDown
 2 | 
 3 | [![CRAN_Status_Badge](http://www.r-pkg.org/badges/version/modelDown)](https://cran.r-project.org/package=modelDown)
 4 | [![DOI](http://joss.theoj.org/papers/10.21105/joss.01444/status.svg)](https://doi.org/10.21105/joss.01444)
 5 | [![R build status](https://github.com/ModelOriented/modelDown/workflows/R-CMD-check/badge.svg)](https://github.com/ModelOriented/modelDown/actions)
 6 | [![Codecov test coverage](https://codecov.io/gh/ModelOriented/modelDown/branch/master/graph/badge.svg)](https://codecov.io/gh/ModelOriented/modelDown?branch=master)
 7 | 
 8 | <a href="https://github.com/ModelOriented/modelDown"><img src="man/figures/modelDownSticker.png" alt="" align="right" width="200px"/></a>
 9 | 
10 | `modelDown` generates a website with HTML summaries for predictive models.
11 | Is uses [DALEX](https://github.com/ModelOriented/DALEX) explainers to compute and plot summaries of how given models behave.  We can see how well models behave (Model Performance, Auditor), how much each variable contributes to predictions (Variable Response) and which variables are the most important for a given model (Variable Importance). We can also compare Concept Drift for pairs of models (Drifter). Additionally, data available on the website can be easily recreated in current R session (using the `archivist` package).
12 | 
13 | `pkgdown` documentation: https://ModelOriented.github.io/modelDown/
14 | 
15 | An example website for regression models: https://mi2datalab.github.io/modelDown_example/
16 | 
17 | ## Getting started
18 | 
19 | Do you want to start right now ? Check out our [getting started](https://ModelOriented.github.io/modelDown/getting-started) guide.
20 | 
21 | Or just simply install it like below:
22 | 
23 | Stable version: `devtools::install_github("ModelOriented/modelDown")`
24 | 
25 | And if you want to get the latest changes:
26 | 
27 | Development version: `devtools::install_github("ModelOriented/modelDown@dev")`
28 | 
29 | ## Contributing
30 | 
31 | If you spot a bug or you have a feature proposal feel free to create an issue in this repository. We are also open to contributions in a form of pull requests. Just follow steps below:
32 | 
33 | 1. Open a new issue (specify an issue type as a label - a bug or an enhancement).
34 | 
35 | Additionally you can:
36 | 
37 | 2. Start a new branch from the `dev` branch. It should be named `bugfix/XX-short-description` or `feature/XX-short-description` where `XX` is an issue number.
38 | 3. Create commits with descriptive messages starting with `#XX`.
39 | 4. Create a pull request of the created branch to the `dev` branch.
40 | 5. Wait for a review from one of the `modelDown` maintainers.
41 | 
42 | Help us build better software!
43 | 
44 | ## Index page
45 | 
46 | <center><img width="600" src="man/figures/index.png"></center>
47 | 
48 | Index page presents basic information about data provided in explainers. You can also see types of all explainers given as parameters. Additionally, summary statistics are available for numerical variables. For categorical variables, tables with frequencies of factor levels are presented.
49 | 
50 | ## Auditor
51 | 
52 | <center><img width="600" src="man/figures/auditor.png"></center>
53 | 
54 | Module shows plots generated by `auditor` package. 
55 | 
56 | ## Drifter
57 | 
58 | Results of `drifter` package are displayed in this tab. In order to see the comparison charts, you have to provide pair of explainers as parameters (for example: `list(explainer_glm_old, explainer_glm_new)`).
59 | 
60 | ## Model Performance
61 | 
62 | <center><img width="600" src="man/figures/performance.png"></center>
63 | 
64 | Module shows result of function `model_performance`. 
65 | 
66 | ## Variable Importance
67 | 
68 | <center><img width="600" src="man/figures/importance.png"></center>
69 | 
70 | Output of function `variable_importance` is presented in form of a plot as well as a table.
71 | 
72 | ## Variable Response
73 | 
74 | <center><img width="600" src="man/figures/response.png"></center>
75 | 
76 | For each variable, plot is created by using function `variable_response`. Plots can be easily navigated using links on the left side. One can provide names of variables to include in the module with argument `vr.vars` (if argument is not used, plots for all variables of first explainer are generated).
77 | 
78 | ## Loading data in R
79 | 
80 | In each tab you can find links with R commands. If you execute them, you can load relevant objects into current R session (`archivist` package is necessary). By default data is stored and loaded from local repository. If you wish to store data on GitHub repository, please provide argument `remote_repository_path`. After generating modelDown website, `repository` folder must be placed under this path.
81 | 
82 | ## Acknowledgments 
83 | 
84 | Work on this package is financially supported by Warsaw University of Technology, Faculty of Mathematics and Information Science.
85 | 


--------------------------------------------------------------------------------
/codecov.yml:
--------------------------------------------------------------------------------
 1 | comment: false
 2 | 
 3 | coverage:
 4 |   status:
 5 |     project:
 6 |       default:
 7 |         target: auto
 8 |         threshold: 1%
 9 |         informational: true
10 |     patch:
11 |       default:
12 |         target: auto
13 |         threshold: 1%
14 |         informational: true
15 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/auditor/generator.R:
--------------------------------------------------------------------------------
  1 | library("DALEX")
  2 | library(ggplot2)
  3 | 
  4 | ROC_HELP_LINK <- "https://modeloriented.github.io/auditor/articles/model_evaluation_audit.html#receiver-operating-characteristic-roc"
  5 | ROC_DOCS_LINK <- "https://modeloriented.github.io/auditor/reference/plotROC.html"
  6 | # LIFT
  7 | LIFT_HELP_LINK <- "https://modeloriented.github.io/auditor/articles/model_evaluation_audit.html#lift-chart"
  8 | LIFT_DOCS_LINK <- "https://modeloriented.github.io/auditor/reference/plotLIFT.html"
  9 | #ACF
 10 | ACF_HELP_LINK <- "https://modeloriented.github.io/auditor/articles/model_residuals_audit.html#plotacf---autocorrelation-function-of-residuals"
 11 | ACF_DOCS_LINK <- "https://modeloriented.github.io/auditor/reference/plotACF.html"
 12 | # RANKING
 13 | RANKING_HELP_LINK <- "https://modeloriented.github.io/auditor/articles/model_performance_audit.html#model-ranking-radar-plot"
 14 | RANKING_DOCS_LINK <- "https://modeloriented.github.io/auditor/reference/plotModelRanking.html"
 15 | # RESIDUALS
 16 | RESIDUALS_HELP_LINK <- "https://modeloriented.github.io/auditor/articles/model_residuals_audit.html#plotresidual---plot-residuals-vs-observed-fitted-or-variable-values"
 17 | RESIDUALS_DOCS_LINK <- "https://modeloriented.github.io/auditor/reference/plotResidual.html"
 18 | # REC
 19 | REC_HELP_LINK <- "https://modeloriented.github.io/auditor/articles/model_residuals_audit.html#plotrec---regression-error-characteristic-rec-curve"
 20 | REC_DOCS_LINK <- "https://modeloriented.github.io/auditor/reference/plotREC.html"
 21 | # RROC
 22 | RROC_HELP_LINK <- "https://modeloriented.github.io/auditor/articles/model_residuals_audit.html#plotrroc---regression-receiver-operating-characteristic-rroc"
 23 | RROC_DOCS_LINK <- "https://modeloriented.github.io/auditor/reference/plotRROC.html"
 24 | # SCALE
 25 | SCALE_HELP_LINK <- "https://modeloriented.github.io/auditor/articles/model_residuals_audit.html#plotscalelocation---scale-location-plot"
 26 | SCALE_DOCS_LINK <- "https://modeloriented.github.io/auditor/reference/plotScaleLocation.html"
 27 | 
 28 | 
 29 | save_plot_image <- function(file_name, models, type, settings){
 30 |   pl <- do.call(plot, c(models, type = type))
 31 |   ggsave(file_name, pl, settings$device)
 32 | }
 33 | 
 34 | make_audit_plot_model <- function(explainers, img_folder, y, options) {
 35 | 
 36 |   models <- lapply(explainers, function(explainer) {
 37 |     explainer
 38 |   })
 39 | 
 40 |   plot_settings <- getPlotSettings(options, "a")
 41 |   extension <- plot_settings$device
 42 | 
 43 |   audit_plots <- list(c(paste("acf", extension, sep = '.'), "ACF"),
 44 |                       c(paste("rroc", extension, sep = '.'), "RROC"),
 45 |                       c(paste("scale_location", extension, sep = '.'), "ScaleLocation"),
 46 |                       c(paste("residuals", extension, sep = '.'), "Residual"),
 47 |                       c(paste("ranking", extension, sep = '.'), "ModelRanking"),
 48 |                       c(paste("rec", extension, sep = '.'), "REC"))
 49 |   # LIFT and ROC only for binary classification
 50 |   if (class(y) == "numeric" && length(levels(as.factor(y))) == 2) {
 51 |     audit_plots <- append(audit_plots, list(c(paste("roc", extension, sep = '.'), "ROC")))
 52 |     audit_plots <- append(audit_plots, list(c(paste("lift", extension, sep = '.'), "LIFT")))
 53 |   } else if (class(y) == "factor") {
 54 |     warning("ROC and LIFT charts are supported only for binary classification.")
 55 |   }
 56 |   result <- list()
 57 |   for(audit_plot in audit_plots) {
 58 |     img_filename <- audit_plot[1]
 59 |     img_path <- file.path(img_folder, img_filename)
 60 | 
 61 |     file.create(img_path)
 62 | 
 63 |     save_plot_image(img_path, models, audit_plot[2], plot_settings)
 64 |     result[audit_plot[2]] <- img_filename
 65 |   }
 66 | 
 67 |   result$link <- save_to_repository(models, options)
 68 | 
 69 |   return(result)
 70 | }
 71 | 
 72 | 
 73 | generator <- function(explainers, options, img_folder) {
 74 | 
 75 |     y = explainers[[1]]$y
 76 |     audit_img_filename <-
 77 |       make_audit_plot_model(explainers, img_folder, y, options)
 78 | 
 79 |     data <- list(
 80 |       # ROC
 81 |       roc_img_filename = audit_img_filename$ROC,
 82 |       ROC_HELP_LINK = ROC_HELP_LINK,
 83 |       ROC_DOCS_LINK = ROC_DOCS_LINK,
 84 |       # LIFT
 85 |       lift_img_filename = audit_img_filename$LIFT,
 86 |       LIFT_HELP_LINK = LIFT_HELP_LINK,
 87 |       LIFT_DOCS_LINK = LIFT_DOCS_LINK,
 88 |       #ACF
 89 |       acf_img_filename = audit_img_filename$ACF,
 90 |       ACF_HELP_LINK = ACF_HELP_LINK,
 91 |       ACF_DOCS_LINK = ACF_DOCS_LINK,
 92 |       # RANKING
 93 |       ranking_img_filename = audit_img_filename$ModelRanking,
 94 |       RANKING_HELP_LINK = RANKING_HELP_LINK,
 95 |       RANKING_DOCS_LINK = RANKING_DOCS_LINK,
 96 |       # RESIDUALS
 97 |       residuals_img_filename = audit_img_filename$Residual,
 98 |       RESIDUALS_HELP_LINK = RESIDUALS_HELP_LINK,
 99 |       RESIDUALS_DOCS_LINK = RESIDUALS_DOCS_LINK,
100 |       # REC
101 |       rec_img_filename = audit_img_filename$REC,
102 |       REC_HELP_LINK = REC_HELP_LINK,
103 |       REC_DOCS_LINK = REC_DOCS_LINK,
104 |       # RROC
105 |       rroc_img_filename = audit_img_filename$RROC,
106 |       RROC_HELP_LINK = RROC_HELP_LINK,
107 |       RROC_DOCS_LINK = RROC_DOCS_LINK,
108 |       # SCALE
109 |       scale_location_img_filename = audit_img_filename$ScaleLocation,
110 |       SCALE_HELP_LINK = SCALE_HELP_LINK,
111 |       SCALE_DOCS_LINK = SCALE_DOCS_LINK,
112 |       archivist_link=audit_img_filename$link
113 |     )
114 | 
115 |   list(
116 |     display_name='Auditor',
117 |     name='auditor',
118 |     data=data
119 |   )
120 | }
121 | 
122 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/auditor/styles.css:
--------------------------------------------------------------------------------
 1 | .data-container {
 2 |   padding: 0px 20px;
 3 | }
 4 | 
 5 | .plot {
 6 |   width: 60%;
 7 | }
 8 | 
 9 | .archivist-code {
10 |   width: 60%;
11 | }
12 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/auditor/template.html:
--------------------------------------------------------------------------------
 1 | <script type="text/javascript">
 2 |   function loadOptionalCharts() {
 3 |     if ("{{lift_img_filename}}" !== "") {
 4 |         document.getElementById("lift").innerHTML =
 5 |         '<img class="plot" src="img/{{lift_img_filename}}" /><br/>\
 6 |          <help-panel help-url="{{LIFT_HELP_LINK}}" docs-url="{{LIFT_DOCS_LINK}}"></help-panel><br />';
 7 |     }
 8 |     if ("{{roc_img_filename}}" !== "") {
 9 |         document.getElementById("roc").innerHTML =
10 |           '<img class="plot" src="img/{{roc_img_filename}}" /><br/>\
11 |            <help-panel help-url="{{ROC_HELP_LINK}}" docs-url="{{ROC_DOCS_LINK}}"></help-panel><br />';
12 |     }
13 |   }
14 | 
15 |   function loadAudit() {
16 |       document.getElementById("auditor_charts").innerHTML =
17 |         '<h2>Model Audit</h2>\
18 |         <p>Get this object: <code>{{archivist_link}}</code></p>\
19 |         <div class="data-container">\
20 |             <div id="lift"></div>\
21 |             <div id="roc"></div>\
22 |             <img class="plot" src="img/{{rroc_img_filename}}" /><br />\
23 |             <help-panel help-url="{{RROC_HELP_LINK}}" docs-url="{{RROC_DOCS_LINK}}"></help-panel>\
24 |             <br/>\
25 |             <img class="plot" src="img/{{ranking_img_filename}}" /><br />\
26 |             <help-panel help-url="{{RANKING_HELP_LINK}}" docs-url="{{RANKING_DOCS_LINK}}"></help-panel>\
27 |             <br/>\
28 |             <img class="plot" src="img/{{residuals_img_filename}}" /><br />\
29 |             <help-panel help-url="{{RESIDUALS_HELP_LINK}}" docs-url="{{RESIDUALS_DOCS_LINK}}"></help-panel>\
30 |             <br/>\
31 |             <img class="plot" src="img/{{rec_img_filename}}" /><br />\
32 |             <help-panel help-url="{{REC_HELP_LINK}}" docs-url="{{REC_DOCS_LINK}}"></help-panel>\
33 |             <br/>\
34 |             <img class="plot" src="img/{{acf_img_filename}}" /><br />\
35 |             <help-panel help-url="{{ACF_HELP_LINK}}" docs-url="{{ACF_DOCS_LINK}}"></help-panel>\
36 |             <br/>\
37 |             <img class="plot" src="img/{{scale_location_img_filename}}" /><br />\
38 |             <help-panel help-url="{{SCALE_HELP_LINK}}" docs-url="{{SCALE_DOCS_LINK}}"></help-panel>\
39 |         </div>'
40 |       loadOptionalCharts()
41 |   }
42 |   $(document).ready(function(){
43 |     loadAudit();
44 |     renderHelpPanels();
45 |   });
46 | </script>
47 | 
48 | <div id="auditor_charts" style="text-align: center"></div>
49 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/drifter/generator.R:
--------------------------------------------------------------------------------
 1 | library("DALEX")
 2 | library(drifter)
 3 | 
 4 | HELP_LINK <- 'https://modeloriented.github.io/drifter/'
 5 | DOCS_LINK <- 'https://modeloriented.github.io/drifter/reference/check_drift.html'
 6 | 
 7 | renderModelDriftTables <- function(data_set, factor_columns){
 8 |   factor_data <- vector()
 9 |   factor_data <- NULL
10 | 
11 |   if(length(factor_columns) > 0) {
12 |     for(i in 1:length(factor_columns)){
13 |       column_number <- factor_columns[[i]]
14 |       column_name <- names(factor_columns)[i]
15 |       temp_table <- kable_styling(kable(table(data_set[, column_number]), col.names = c(column_name, "Frequency")), bootstrap_options = c("responsive", "bordered", "hover"),full_width = FALSE)
16 |       factor_data <- paste(factor_data, temp_table, sep="<br>")
17 |     }
18 |   }
19 | 
20 |   return(factor_data)
21 | }
22 | 
23 | renderTable <- function(data_table) {
24 |   return(kable_styling(kable(data_table), bootstrap_options = c("responsive", "bordered", "hover"), full_width = FALSE))
25 | }
26 | 
27 | renderDrifterSection <- function(section_name, data_table) {
28 |   section_header <- paste0("<h3 class='section-label'>", section_name, "</h3>")
29 |   section_data <- renderTable(data_table)
30 |   return(paste0("<div class='column'>", section_header, section_data, "</div>"))
31 | }
32 | 
33 | generator <- function(explainer_pairs, options, img_folder) {
34 | 
35 |   drifter_data <- ""
36 |   for(pair in explainer_pairs) {
37 |     old_explainer <- pair[[1]]
38 |     new_explainer <- pair[[2]]
39 |     drifter_data <- paste0(drifter_data, "<div class='row'>", "<h3 class='model-label'>", old_explainer$label, "</h3>")
40 |     drift <- check_drift(old_explainer$model, new_explainer$model,
41 |                          old_explainer$data, new_explainer$data,
42 |                          old_explainer$y, new_explainer$y,
43 |                          predict_function = old_explainer$predict_function)
44 | 
45 |     archivist_link <- save_to_repository(drift, options)
46 | 
47 |     model_drift <- renderDrifterSection("Model Drift", drift$model_drift)
48 |     covariate_drift <- renderDrifterSection("Covariate Drift", data.frame(Variable = drift$covariate_drift$variables , Drift = drift$covariate_drift$drift))
49 |     residual_drift <- renderDrifterSection("Residual Drift", drift$residual_drift)
50 |     help_section <- paste0("<help-panel help-url='", HELP_LINK, "' docs-url='", DOCS_LINK, "'></help-panel>")
51 |     archivist_section <- paste0("<div class='archivist-code'><p>Get this object: <code>", archivist_link, "</code></p></div>")
52 |     drifter_data <- paste0(drifter_data, model_drift, covariate_drift, residual_drift, help_section, "</div>", archivist_section)
53 |   }
54 | 
55 |   list(
56 |     display_name='Drifter',
57 |     name='drifter',
58 |     data=list(drifter_data = drifter_data)
59 |   )
60 | }
61 | 
62 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/drifter/styles.css:
--------------------------------------------------------------------------------
 1 | .column {
 2 |   float: left;
 3 |   width: 33%;
 4 | }
 5 | 
 6 | .row:after {
 7 |   content: "";
 8 |   display: table;
 9 |   clear: both;
10 | }
11 | 
12 | .model-label {
13 |   font-size: 200%;
14 | }
15 | 
16 | .section-label {
17 |   font-size: 120%;
18 | }
19 | 
20 | .archivist-code {
21 |   margin: auto;
22 | }
23 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/drifter/template.html:
--------------------------------------------------------------------------------
1 | <div class="text-center">
2 |     {{{drifter_data}}}
3 | </div>
4 | 
5 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/model_performance/generator.R:
--------------------------------------------------------------------------------
 1 | library("DALEX")
 2 | library(ggplot2)
 3 | 
 4 | HELP_LINK <- "https://pbiecek.github.io/ema/modelPerformance.html"
 5 | DOCS_LINK <- "https://modeloriented.github.io/DALEX/reference/model_performance.html"
 6 | 
 7 | save_plot_image <- function(file_name, models, height, img_folder, settings){
 8 | 
 9 |   path <- file.path(img_folder, file_name)
10 |   file.create(path)
11 | 
12 |   plot_obj <- do.call(plot, models) + theme(text = element_text(size=settings$font_size))
13 |   ggsave(path, plot_obj, settings$device)
14 | }
15 | 
16 | make_model_performance_plot_model <- function(explainers, img_folder, options) {
17 |   plot_settings <- getPlotSettings(options, "mp")
18 | 
19 |   img_filename <- paste('model_performance', plot_settings$device, sep = '.')
20 |   img_box_filename <- paste('model_performance_box', plot_settings$device, sep = '.')
21 | 
22 |   models <- lapply(explainers, function(explainer) {
23 |     model_performance(explainer)
24 |   })
25 | 
26 |   save_plot_image(img_filename, models, 5, img_folder, plot_settings)
27 |   link_mp <- save_to_repository(models, options)
28 | 
29 |   models$geom <- "boxplot"
30 |   save_plot_image(img_box_filename, models, 4, img_folder, plot_settings)
31 |   link_mp_box <- save_to_repository(models, options)
32 | 
33 |   list(img_filename = img_filename,
34 |        img_box_filename = img_box_filename,
35 |        archivist_link_mp1 = link_mp,
36 |        archivist_link_mp2 = link_mp_box)
37 | }
38 | 
39 | generator <- function(explainers, options, img_folder) {
40 | 
41 |   img_filename <- make_model_performance_plot_model(explainers, img_folder, options)
42 | 
43 |   list(
44 |     display_name='Model Performance',
45 |     name='model_performance',
46 |     data=list(
47 |       img_filename_mp1 = img_filename$img_filename,
48 |       img_filename_mp2 = img_filename$img_box_filename,
49 |       HELP_LINK = HELP_LINK,
50 |       DOCS_LINK = DOCS_LINK,
51 |       archivist_link_mp1 = img_filename$archivist_link_mp1,
52 |       archivist_link_mp2 = img_filename$archivist_link_mp2
53 |     )
54 |   )
55 | }
56 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/model_performance/styles.css:
--------------------------------------------------------------------------------
 1 | .data-container {
 2 |   padding: 0px 20px;
 3 | }
 4 | 
 5 | .plot {
 6 |   width: 60%;
 7 | }
 8 | 
 9 | .archivist-code {
10 |   width: 60%;
11 |   margin: auto;
12 | }
13 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/model_performance/template.html:
--------------------------------------------------------------------------------
 1 | <div class="text-center">
 2 |     <h2>Model Performance</h2>
 3 |     <div class="data-container">
 4 |         <div>
 5 |           <img class="plot" src="img/{{img_filename_mp1}}" />
 6 |           <br >
 7 |           <help-panel help-url="{{HELP_LINK}}" docs-url="{{DOCS_LINK}}"></help-panel>
 8 |           <div class="archivist-code"><p>Get this object: <code>{{archivist_link_mp1}}</code></p></div>
 9 |         </div>
10 |         <br/>
11 |         <div>
12 |           <img class="plot" src="img/{{img_filename_mp2}}" />
13 |           <br >
14 |           <help-panel help-url="{{HELP_LINK}}" docs-url="{{DOCS_LINK}}"></help-panel>
15 |           <div class="archivist-code"><p>Get this object: <code>{{archivist_link_mp2}}</code></p></div>
16 |         </div>
17 |     </div>
18 | </div>
19 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/variable_importance/generator.R:
--------------------------------------------------------------------------------
 1 | library(DALEX)
 2 | library(ggplot2)
 3 | library(kableExtra)
 4 | 
 5 | HELP_LINK <- "https://pbiecek.github.io/ema/featureImportance.html"
 6 | DOCS_LINK <- "https://modeloriented.github.io/DALEX/reference/variable_importance.html"
 7 | 
 8 | save_plot_image <- function(file_name, models, settings){
 9 |   pl <- do.call(plot, models) + theme(text = element_text(size=settings$font_size))
10 |   ggsave(file_name, pl, settings$device)
11 | }
12 | 
13 | make_variable_importance_table <- function(explainers) {
14 |   importance_table <- vector()
15 |   importance_table <- NULL
16 | 
17 |   importance_table_list <- list()
18 |   for (explainer_id in seq_along(explainers)) {
19 |     explainer <- explainers[[explainer_id]]
20 |     model <- variable_importance(explainer, type = "raw")
21 |     # only first permutation (from DALEX 1.0)
22 |     model <- model[model$permutation == 0,]
23 |     importance_table_list[[explainer_id]] <- model[,c(1,3)]
24 |   }
25 |   importance_table <- do.call(rbind, importance_table_list)
26 | 
27 |   html <- kable_styling(kable(importance_table, row.names = FALSE),
28 |                         bootstrap_options = c("responsive", "hover"),
29 |                         full_width = FALSE)
30 |   i <- 1
31 |   for(explainer_id in seq_along(explainers)){
32 |     explainer <- explainers[[explainer_id]]
33 |     label <- explainer$label
34 |     html <- group_rows(html, label, i, i + nrow(importance_table_list[[explainer_id]]) - 1,
35 |                        hline_before = TRUE, hline_after = TRUE)
36 |     i <- i + nrow(importance_table_list[[explainer_id]])
37 |   }
38 | 
39 |   html
40 | }
41 | 
42 | create_plot_image <- function(models, img_folder, options){
43 |   plot_settings <- getPlotSettings(options, "vi")
44 | 
45 |   img_filename <- paste('variable_importance', plot_settings$device, sep='.')
46 |   img_path <- file.path(img_folder, img_filename)
47 | 
48 |   file.create(img_path)
49 |   save_plot_image(img_path, models, plot_settings)
50 | 
51 |   img_filename
52 | }
53 | 
54 | generate_models <- function(explainers){
55 |   models <- lapply(explainers, function(explainer) {
56 |     model <- variable_importance(explainer, type="raw")
57 |     # only first permutation (from DALEX 1.0)
58 |     model[model$permutation == 0,]
59 |   })
60 |   models
61 | }
62 | 
63 | generator <- function(explainers, options, img_folder) {
64 | 
65 |   variable_importance_table <- make_variable_importance_table(explainers)
66 | 
67 |   models <- generate_models(explainers)
68 | 
69 |   filename <- create_plot_image(models, img_folder, options)
70 | 
71 |   link <- save_to_repository(models, options)
72 | 
73 |   list(
74 |     display_name='Variable Importance',
75 |     name='variable_importance',
76 |     data=list(
77 |       img_filename=filename,
78 |       dataframe=variable_importance_table,
79 |       archivist_link = link,
80 |       HELP_LINK = HELP_LINK,
81 |       DOCS_LINK = DOCS_LINK
82 |     )
83 |   )
84 | }
85 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/variable_importance/styles.css:
--------------------------------------------------------------------------------
 1 | .data-container {
 2 |   padding: 0px 20px;
 3 | }
 4 | 
 5 | .plot {
 6 |   width: 60%;
 7 | }
 8 | 
 9 | .archivist-code {
10 |   width: 60%;
11 |   margin: auto;
12 | }
13 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/variable_importance/template.html:
--------------------------------------------------------------------------------
 1 | <div class="text-center">
 2 |     <div class="data-container">
 3 |       <h2>Feature Importance</h2>
 4 |       <img class="plot" src="img/{{img_filename}}" />
 5 |       <br />
 6 |       <help-panel help-url="{{HELP_LINK}}" docs-url="{{DOCS_LINK}}"></help-panel>
 7 |       <div class="archivist-code"><p>Get this object: <code>{{archivist_link}}</code></p></div>
 8 |     </div>
 9 |     <div class="data-container">
10 |       <br>
11 |       {{{dataframe}}}
12 |     </div>
13 | </div>
14 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/variable_response/generator.R:
--------------------------------------------------------------------------------
 1 | library("DALEX")
 2 | library(ggplot2)
 3 | 
 4 | HELP_LINK <- "https://pbiecek.github.io/ema/partialDependenceProfiles.html"
 5 | DOCS_LINK <- "https://modeloriented.github.io/DALEX/reference/model_profile.html"
 6 | 
 7 | save_plot_image <- function(file_name, models, settings){
 8 |   pl <- do.call(plot, models) + theme(text = element_text(size=settings$font_size))
 9 |   ggsave(file_name, pl, settings$device)
10 | }
11 | 
12 | make_variable_plot <- function(variable_name, types, models, img_folder, options) {
13 |   plot_settings <- getPlotSettings(options, "vr")
14 |   img_filename <- paste('variable_response_', variable_name, '_', paste(types, collapse=''), '.', plot_settings$device, sep='')
15 | 
16 |   img_path <- file.path(img_folder, img_filename)
17 | 
18 |   file.create(img_path)
19 |   save_plot_image(img_path, models, plot_settings)
20 | 
21 |   return(img_filename)
22 | }
23 | 
24 | make_variable_plot_model <- function(variable_name, explainers, img_folder, options) {
25 | 
26 |   types <- options[["vr.type"]]
27 |   if(is.null(types)) {
28 |     types <- "partial"
29 |   }
30 | 
31 |   models_per_type <- lapply(types, function(type) {
32 |     lapply(explainers, function(explainer) { model_profile(explainer, variable_name, type=type)$agr_profiles })
33 |   })
34 | 
35 |   models <- do.call(c, models_per_type)
36 | 
37 |   plot_filename <- make_variable_plot(variable_name, types, models, img_folder, options)
38 | 
39 |   link <- save_to_repository(models, options)
40 | 
41 |   list(
42 |     variable_name=variable_name,
43 |     img_filename=plot_filename,
44 |     archivist_link = link
45 |   )
46 | }
47 | 
48 | sort_by_importance <- function(explainers, variables) {
49 |   importance <- variable_importance(explainers[[1]])
50 |   # only first permutation (from DALEX 1.0)
51 |   importance <- importance[importance$permutation == 0,]
52 |   variable_dropouts <- importance$dropout_loss[
53 |     sapply(variables, function(var_name) {
54 |       index <- which(importance$variable == var_name)
55 |     })]
56 |   variables_order <- order(variable_dropouts, decreasing = TRUE)
57 |   sorted_variables <- variables[variables_order]
58 |   return(sorted_variables)
59 | }
60 | 
61 | generator <- function(explainers, options, img_folder, sort_by_importance = TRUE) {
62 | 
63 |   variables <- options[["vr.vars"]]
64 |   if(is.null(variables)) {
65 |     variables <- colnames(explainers[[1]]$data)
66 |   }
67 |   # how to sort variables?
68 |   if (sort_by_importance) {
69 |     variables <- sort_by_importance(explainers, variables)
70 |   } else {
71 |     variables <- sort(variables)
72 |   }
73 | 
74 |   variable_models <- lapply(variables, make_variable_plot_model, explainers, img_folder, options)
75 | 
76 |   list(
77 |     display_name='Variable Response',
78 |     name='variable_response',
79 |     data=list(
80 |       variables=variable_models,
81 |       HELP_LINK = HELP_LINK,
82 |       DOCS_LINK = DOCS_LINK
83 |     )
84 |   )
85 | }
86 | 
87 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/variable_response/styles.css:
--------------------------------------------------------------------------------
 1 | .variable-selector-panel {
 2 |   position: fixed;
 3 |   width: 300px;
 4 | }
 5 | 
 6 | .variable-link {
 7 |   padding: 5px 15px;
 8 | }
 9 | 
10 | .main-content {
11 |   left: 300px;
12 |   width: auto;
13 | }
14 | 
15 | .plot-container {
16 |   padding: 0px 20px;
17 | /*  text-align: center; */
18 | }
19 | 
20 | .plot {
21 |   width: 60%;
22 | }
23 | 
24 | .archivist-code {
25 |   width: 60%;
26 | }
27 | 
28 | .contents h1, .contents h2, .contents h3, .contents h4 {
29 |   padding-top: 50px;
30 |   margin-top: -50px;
31 | }
32 | 


--------------------------------------------------------------------------------
/inst/extdata/modules/variable_response/template.html:
--------------------------------------------------------------------------------
 1 | <div class="variable-selector-panel">
 2 |     <b>Features</b>
 3 |     <div class="panel panel-default">
 4 |         <ul class="list-group">
 5 |             {{#variables}}
 6 |             <li class="list-group-item variable-link">
 7 |                 <a href="#{{variable_name}}">{{variable_name}}</a>
 8 |             </li>
 9 |             {{/variables}}
10 |         </ul>
11 |     </div>
12 | </div>
13 | <div class="col-md-push-3 col-md-9 main-content contents">
14 |     {{#variables}}
15 |     <div class="plot-container">
16 |         <h2 id="{{variable_name}}" class="hasAnchor">
17 |             <a href="#{{variable_name}}" class="anchor"></a>
18 |             Feature: {{variable_name}}
19 |         </h2>
20 |         <img class="plot" src="img/{{img_filename}}" />
21 |         <br />
22 |         <help-panel help-url="{{HELP_LINK}}" docs-url="{{DOCS_LINK}}"></help-panel>
23 |         <div class="archivist-code"><p>Get this object: <code>{{archivist_link}}</code></p></div>
24 |     </div>
25 |     {{/variables}}
26 | </div>
27 | 


--------------------------------------------------------------------------------
/inst/extdata/template/base_template.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html>
  3 | 
  4 | <head>
  5 |   <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
  6 |   <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u"
  7 |     crossorigin="anonymous">
  8 |   <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa"
  9 |     crossorigin="anonymous"></script>
 10 |   <link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.6.3/css/font-awesome.min.css" rel="stylesheet" integrity="sha384-T8Gy5hrqNKT+hzMclPo118YTQO6cYprQmhrYwIiQ/3axmI1hQomh7Ud2hPOy8SP1"
 11 |     crossorigin="anonymous">
 12 | 
 13 |   <link rel="stylesheet" type="text/css" href="{{root_path}}mi2template.css">
 14 |   <link rel="stylesheet" type="text/css" href="{{root_path}}style.css">
 15 |   {{#extra_css}}
 16 |   <link rel="stylesheet" type="text/css" href="{{.}}">
 17 |   {{/extra_css}}
 18 | 
 19 |   <script>
 20 |     function makeHelpPanel(help_link, docs_link) {
 21 |       return '\
 22 |             <div class="help-panel">\
 23 |               <div class="title"><i class="fa fa-question" aria-hidden="true"></i> Helpful resources</div>\
 24 |               <a href="' + docs_link + '" target="_blank">Technical documentation</a>\
 25 |               <a href="' + help_link + '" target="_blank">Business description</a>\
 26 |             </div>\
 27 |             ';
 28 |     };
 29 | 
 30 |     function renderHelpPanels() {
 31 |       var panel = document.getElementsByTagName("help-panel")[0];
 32 | 
 33 |       while(panel) {
 34 |         var docs_url = panel.attributes["docs-url"].value; 
 35 |         var help_url = panel.attributes["help-url"].value;
 36 |         
 37 |         var html = makeHelpPanel(help_url, docs_url);
 38 |         panel.outerHTML = html;
 39 | 
 40 |         panel = document.getElementsByTagName("help-panel")[0]; 
 41 |         // Not iterating the result because it changes after each iteration
 42 |       }
 43 |     }
 44 | 
 45 |     $(document).ready(function(){
 46 |       renderHelpPanels();
 47 |     });
 48 |   </script>
 49 | </head>
 50 | 
 51 | <body>
 52 | 
 53 |   <div class="container template-home">
 54 | 
 55 |     <header>
 56 |       <div class="navbar navbar-default navbar-fixed-top navbar-mi2" role="navigation">
 57 |         <div class="container">
 58 |           <div class="navbar-header">
 59 |             <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar">
 60 |               <span class="icon-bar"></span>
 61 |               <span class="icon-bar"></span>
 62 |               <span class="icon-bar"></span>
 63 |             </button>
 64 |             </a>
 65 |             <a class="navbar-brand navbar-mi2" href="{{root_path}}index.html">modelDown</a>
 66 |           </div>
 67 |           <div id="navbar" class="navbar-collapse collapse">
 68 |             <ul class="navbar-mi2 nav navbar-nav">
 69 |               <li>
 70 |                 <a href="{{root_path}}index.html">
 71 |                   <span class="fa fa-home fa-lg"></span>
 72 | 
 73 |                 </a>
 74 |               </li>
 75 | 
 76 |               {{#menu_links}}
 77 |               <li>
 78 |                   <a href="{{root_path}}{{link}}">{{name}}</a>
 79 |               </li>
 80 |               {{/menu_links}}
 81 |             </ul>
 82 |             <ul class="nav navbar-nav navbar-right"></ul>
 83 |           </div>
 84 |           <!--/.nav-collapse -->
 85 |         </div>
 86 |         <!--/.container -->
 87 |       </div>
 88 |       <!--/.navbar -->
 89 |     </header>
 90 | 
 91 |     <div class="row" style="position: relative; margin-bottom:60px;">
 92 |       {{&content}}
 93 |     </div>
 94 | 
 95 | 
 96 |   <footer>
 97 |     <div class="footer">
 98 |       <div class="text-center">Site built with <a href="https://github.com/MI2DataLab/modelDown">modelDown</a>.</div>
 99 |       <div class="text-center" style="margin: 0px 0px 5px;">Last updated: {{datetime}} (session_info: <a href="session_info/session_info.txt">.txt</a>, <a href="session_info/session_info.rda">.rda</a>)</div>
100 |     </div>
101 |   </footer>
102 | 
103 |   </div>
104 | 
105 | </body>
106 | </html>
107 | 


--------------------------------------------------------------------------------
/inst/extdata/template/index_template.html:
--------------------------------------------------------------------------------
 1 | <header class="intro-header">
 2 |   <div class="container text-center">
 3 |     <h1>modelDown</h1>
 4 |     <p class="lead">Explore your model!</p>
 5 |   </div>
 6 | </header>
 7 | 
 8 | 
 9 | <br>
10 | <br>
11 | 
12 | <div class="row">
13 |   <div class="col-md-8">
14 |     <h3 class="my-3">Basic data information</h3>
15 |     <ul>
16 |       <li>{{observations_count}} observations</li>
17 |       <li>{{columns_count}} columns</li>
18 |     </ul>
19 |   </div>
20 |   <div class="col-md-4" >
21 |     <h3 class="my-3">Explainers</h3>
22 |       {{{explainers}}}
23 |   </div>
24 | </div>
25 | 
26 | <hr>
27 | 
28 | 
29 | <div class="text-center">
30 |   <h3 class="my-3">Summaries for numerical variables</h3>
31 |   <br>
32 |   {{{data_summary}}}
33 |   <hr>
34 |   {{#factor_summary}}
35 |     <h3 class="my-3">Summaries for categorical variables</h3>
36 |     {{{factor_summary}}}
37 |   {{/factor_summary}}
38 | </div>
39 | 
40 | <br>
41 | <br>
42 | 


--------------------------------------------------------------------------------
/inst/extdata/template/link.svg:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="utf-8"?>
 2 | <!-- Generator: Adobe Illustrator 19.2.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
 3 | <svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
 4 | 	 viewBox="0 0 20 20" style="enable-background:new 0 0 20 20;" xml:space="preserve">
 5 | <style type="text/css">
 6 | 	.st0{fill:#75AADB;}
 7 | </style>
 8 | <path class="st0" d="M4,11.3h1.3v1.3H4c-2,0-4-2.3-4-4.7s2.1-4.7,4-4.7h5.3c1.9,0,4,2.3,4,4.7c0,1.9-1.2,3.6-2.7,4.3v-1.5
 9 | 	C11.4,10.2,12,9.1,12,8c0-1.7-1.4-3.3-2.7-3.3H4C2.7,4.7,1.3,6.3,1.3,8S2.7,11.3,4,11.3z M16,7.3h-1.3v1.3H16c1.3,0,2.7,1.6,2.7,3.3
10 | 	s-1.4,3.3-2.7,3.3h-5.3C9.4,15.3,8,13.7,8,12c0-1.1,0.6-2.2,1.3-2.8V7.7C7.9,8.4,6.7,10.1,6.7,12c0,2.4,2.1,4.7,4,4.7H16
11 | 	c1.9,0,4-2.3,4-4.7S18,7.3,16,7.3z"/>
12 | </svg>
13 | 


--------------------------------------------------------------------------------
/inst/extdata/template/mi2template.css:
--------------------------------------------------------------------------------
  1 | /* Sticker footer */
  2 | body > .container {
  3 |   display: flex;
  4 |   padding-top: 60px;
  5 |   min-height: calc(100vh);
  6 |   flex-direction: column;
  7 | }
  8 | 
  9 | body > .container .row {
 10 |   flex: 1;
 11 | }
 12 | 
 13 | footer {
 14 |   margin-top: 45px;
 15 |   padding: 35px 0 36px;
 16 |   border-top: 1px solid #e5e5e5;
 17 |   color: #666;
 18 |   display: flex;
 19 | }
 20 | footer p {
 21 |   margin-bottom: 0;
 22 | }
 23 | footer div {
 24 |   flex: 1;
 25 | }
 26 | footer .pkgdown {
 27 |   text-align: right;
 28 | }
 29 | footer p {
 30 |   margin-bottom: 0;
 31 | }
 32 | 
 33 | img.icon {
 34 |   float: right;
 35 | }
 36 | 
 37 | img {
 38 |   max-width: 100%;
 39 | }
 40 | 
 41 | /* Section anchors ---------------------------------*/
 42 | 
 43 | a.anchor {
 44 |   margin-left: -30px;
 45 |   display:inline-block;
 46 |   width: 30px;
 47 |   height: 30px;
 48 |   visibility: hidden;
 49 | 
 50 |   background-image: url(./link.svg);
 51 |   background-repeat: no-repeat;
 52 |   background-size: 20px 20px;
 53 |   background-position: center center;
 54 | }
 55 | 
 56 | .hasAnchor:hover a.anchor {
 57 |   visibility: visible;
 58 | }
 59 | 
 60 | @media (max-width: 767px) {
 61 |   .hasAnchor:hover a.anchor {
 62 |     visibility: hidden;
 63 |   }
 64 | }
 65 | 
 66 | 
 67 | /* Fixes for fixed navbar --------------------------*/
 68 | 
 69 | .contents h1, .contents h2, .contents h3, .contents h4 {
 70 |   padding-top: 60px;
 71 |   margin-top: -60px;
 72 | }
 73 | 
 74 | /* Static header placement on mobile devices */
 75 | @media (max-width: 767px) {
 76 |   .navbar-fixed-top {
 77 |     position: absolute;
 78 |   }
 79 |   .navbar {
 80 |     padding: 0;
 81 |   }
 82 | }
 83 | 
 84 | 
 85 | /* Sidebar --------------------------*/
 86 | 
 87 | #sidebar {
 88 |   margin-top: 30px;
 89 | }
 90 | #sidebar h2 {
 91 |   font-size: 1.5em;
 92 |   margin-top: 1em;
 93 | }
 94 | 
 95 | #sidebar h2:first-child {
 96 |   margin-top: 0;
 97 | }
 98 | 
 99 | #sidebar .list-unstyled li {
100 |   margin-bottom: 0.5em;
101 | }
102 | 
103 | /* Reference index & topics ----------------------------------------------- */
104 | 
105 | .ref-index th {font-weight: normal;}
106 | .ref-index h2 {font-size: 20px;}
107 | 
108 | .ref-index td {vertical-align: top;}
109 | .ref-index .alias {width: 40%;}
110 | .ref-index .title {width: 60%;}
111 | 
112 | .ref-index .alias {width: 40%;}
113 | .ref-index .title {width: 60%;}
114 | 
115 | .ref-arguments th {text-align: right; padding-right: 10px;}
116 | .ref-arguments th, .ref-arguments td {vertical-align: top;}
117 | .ref-arguments .name {width: 20%;}
118 | .ref-arguments .desc {width: 80%;}
119 | 
120 | /* Nice scrolling for wide elements --------------------------------------- */
121 | 
122 | table {
123 |   display: block;
124 |   overflow: auto;
125 | }
126 | 
127 | /* Syntax highlighting ---------------------------------------------------- */
128 | 
129 | pre {
130 |   word-wrap: normal;
131 |   word-break: normal;
132 |   border: 1px solid #eee;
133 | }
134 | 
135 | pre, code {
136 |   background-color: #f8f8f8;
137 |   color: #333;
138 | }
139 | 
140 | pre .img {
141 |   margin: 5px 0;
142 | }
143 | 
144 | pre .img img {
145 |   background-color: #fff;
146 |   display: block;
147 |   height: auto;
148 | }
149 | 
150 | code a, pre a {
151 |   color: #375f84;
152 | }
153 | table {
154 |     display: block;
155 |     overflow: auto;
156 | 	width: 100% !important;
157 | }
158 | 
159 | .fl      {color: #1514b5;}
160 | .fu      {color: #000000;} /* function */
161 | .ch,.st  {color: #036a07;} /* string */
162 | .kw      {color: #264D66;} /* keyword */
163 | .co      {color: #888888;} /* comment */
164 | 
165 | .message { color: black;   font-weight: bolder;}
166 | .error   { color: orange;  font-weight: bolder;}
167 | .warning { color: #6A0366; font-weight: bolder;}
168 | 
169 | .navbar-mi2logo {
170 |     float: left;
171 | 	margin-right: 15px;
172 | 	margin-top: 2px;
173 | }
174 | .navbar-mi2 {
175 | 	background-color: #4a3c89;
176 |     color: #fff !important;
177 | 	margin-right: 0px;
178 | }
179 | .navbar-mi2 > li > a {
180 |     color: #fff !important;
181 | }
182 | .navbar-mi2 > .active > a{
183 |     background-color: #370f54 !important;
184 | }
185 | .navbar-mi2 > .open > a:focus, .nav-pills> .open > a:focus{
186 |     background-color: #370f54 !important;
187 | }
188 | .dropdown-menu > .active > a, .dropdown-menu > .active > a:focus{
189 |     background-color: #370f54 !important;
190 | }
191 | 
192 |  .contents-mi2 > li > a:focus, .nav-pills > li > a:focus {
193 |     background-color: #4a3c89 !important;
194 | 	color: #fff;
195 | }
196 |  .contents-mi2 > li.active > a, .nav-pills > li.active > a{
197 |     background-color: #370f54 !important;
198 | }
199 |  .contents-mi2 > li > a, .nav-pills > li > a{
200 |     background-color: #4a3c89 !important;
201 | 	color: #fff;
202 | }
203 | 
204 | .sidebar-logo {
205 | 	display:block;
206 | 	margin-left:auto;
207 | 	margin-right:auto;
208 | 	text-align: justify;
209 | }


--------------------------------------------------------------------------------
/inst/extdata/template/style.css:
--------------------------------------------------------------------------------
 1 | body {font-family: Arial;}
 2 | 
 3 | table {
 4 |   display: table;
 5 | }
 6 | 
 7 | .intro-header {
 8 |   border-radius: 10px;
 9 |   background-color: #6d60a9;
10 |   box-shadow: 5px 5px 15px -5px #444;
11 |   color: #fff;
12 | }
13 | 
14 | .footer {
15 |     width: 100%;
16 |     background-color: #4a3c89;
17 |     text-align: center;
18 |     color: white;
19 |     padding: 10px 20px;
20 |     border-radius: 10px;
21 |     box-shadow: 5px 5px 15px -5px #444;
22 | }
23 | 
24 | .footer a {
25 |   color: #c8c3dc;
26 | }
27 | 
28 | .footer a:hover {
29 |   color: #aea7cc;
30 | }
31 | 
32 | /* Help panel */
33 | .help-panel {
34 |   margin-top: 10px;
35 |   color: #333;
36 |   display: inline-block;
37 | }
38 | 
39 | .help-panel > * {
40 |     display: inline-block;
41 |     padding: 1px 10px;
42 |     position: relative;
43 |     float: left;
44 |     color: #333;
45 |     font-size: 12px;
46 |     font-weight: 400;
47 |     line-height: 1.5;
48 |     text-align: center;
49 |     white-space: nowrap;
50 |     vertical-align: middle;
51 |     background-color: #fff;
52 |     border: 1px solid #ccc;
53 |     border-radius: 3px;
54 | }
55 | 
56 | .help-panel > a {
57 |     touch-action: manipulation;
58 |     cursor: pointer;
59 |     user-select: none;
60 | }
61 | 
62 | .help-panel > a:hover {
63 |     background-color: #e6e6e6;
64 |     border-color: #adadad;
65 |     text-decoration: none;
66 |     color: #333;
67 | }
68 | 
69 | .help-panel > *:first-child:not(:last-child) {
70 |     border-top-right-radius: 0;
71 |     border-bottom-right-radius: 0;
72 | }
73 | 
74 | .help-panel > *:not(:first-child):not(:last-child) {
75 |     border-radius: 0;
76 | }
77 | 
78 | .help-panel *+* {
79 |     margin-left: -1px;
80 | }
81 | 
82 | .help-panel > .title {
83 |   color: #888;
84 | }
85 | 
86 | .help-panel > .title > i.fa {
87 |   font-size: 13px;
88 | }
89 | 


--------------------------------------------------------------------------------
/man/figures/auditor.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/man/figures/auditor.png


--------------------------------------------------------------------------------
/man/figures/importance.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/man/figures/importance.png


--------------------------------------------------------------------------------
/man/figures/index.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/man/figures/index.png


--------------------------------------------------------------------------------
/man/figures/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/man/figures/logo.png


--------------------------------------------------------------------------------
/man/figures/modelDownSticker.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/man/figures/modelDownSticker.png


--------------------------------------------------------------------------------
/man/figures/performance.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/man/figures/performance.png


--------------------------------------------------------------------------------
/man/figures/response.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/man/figures/response.png


--------------------------------------------------------------------------------
/man/modelDown.Rd:
--------------------------------------------------------------------------------
 1 | % Generated by roxygen2: do not edit by hand
 2 | % Please edit documentation in R/modelDown.R
 3 | \name{modelDown}
 4 | \alias{modelDown}
 5 | \title{Generates a website with HTML summaries for predictive models}
 6 | \usage{
 7 | modelDown(
 8 |   ...,
 9 |   modules = c("auditor", "drifter", "model_performance", "variable_importance",
10 |     "variable_response"),
11 |   output_folder = "output",
12 |   repository_name = "repository",
13 |   should_open_website = interactive()
14 | )
15 | }
16 | \arguments{
17 | \item{...}{one or more explainers created with \code{DALEX::explain()} function. Pair of explainer could be provided to check drift of models}
18 | 
19 | \item{modules}{modules that should be included in the website}
20 | 
21 | \item{output_folder}{folder where the website will be saved}
22 | 
23 | \item{repository_name}{name of local archivist repository that will be created}
24 | 
25 | \item{should_open_website}{should generated website be automatically opened in default browser}
26 | }
27 | \description{
28 | Generates a website with HTML summaries for predictive models
29 | }
30 | \details{
31 | Additional arguments that could by passed by name:
32 | \itemize{
33 |   \item{remote_repository_path} {Path to remote repository that stores folder with archivist repository. If not provided, links to local repository will be shown.}
34 |   \item{device} {Device to use. Tested for "png" and "svg", but values from \code{ggplot2::ggsave} function should be working fine. Defaults to "png".}
35 |   \item{vr.vars} {variables which will be examined in Variable Response module. Defaults to all variables. Example vr.vars = c("var1", "var2")}
36 |   \item{vr.type} {types of examinations which will be conducteed in Variable Response module. Defaults to "partial". Example vr.type = c("partial", "conditional", "accumulated")}
37 | }
38 | }
39 | \examples{
40 | if(FALSE){
41 | require("ranger")
42 | require("breakDown")
43 | require("DALEX")
44 | 
45 | 
46 | # Generate simple modelDown page
47 | HR_data_selected <- HR_data[1000:3000,]
48 | HR_glm_model <- glm(left~., HR_data_selected, family = "binomial")
49 | explainer_glm <- explain(HR_glm_model, data=HR_data_selected, y = HR_data_selected$left)
50 | 
51 | modelDown::modelDown(explainer_glm,
52 |                      modules = c("model_performance", "variable_importance",
53 |                                  "variable_response"),
54 |                      output_folder = tempdir(),
55 |                      repository_name = "HR",
56 |                      device = "png",
57 |                      vr.vars= c("average_montly_hours"),
58 |                      vr.type = "partial")
59 | 
60 | # More complex example with all modules
61 | HR_ranger_model <- ranger(as.factor(left) ~ .,
62 |                       data = HR_data, num.trees = 500, classification = TRUE, probability = TRUE)
63 | explainer_ranger <- explain(HR_ranger_model,
64 |                       data = HR_data, y = HR_data$left, function(model, data) {
65 |  return(predict(model, data)$prediction[,2])
66 | }, na.rm=TRUE)
67 | 
68 | # Two glm models used for drift detection
69 | HR_data1 <- HR_data[1:4000,]
70 | HR_data2 <- HR_data[4000:nrow(HR_data),]
71 | HR_glm_model1 <- glm(left~., HR_data1, family = "binomial")
72 | HR_glm_model2 <- glm(left~., HR_data2, family = "binomial")
73 | explainer_glm1 <- explain(HR_glm_model1, data=HR_data1, y = HR_data1$left)
74 | explainer_glm2 <- explain(HR_glm_model2, data=HR_data2, y = HR_data2$left)
75 | 
76 | modelDown::modelDown(list(explainer_glm1, explainer_glm2),
77 |   modules = c("auditor", "drifter", "model_performance", "variable_importance",
78 |               "variable_response"),
79 |   output_folder = tempdir(),
80 |   repository_name = "HR",
81 |   remote_repository_path = "some_user/remote_repo_name",
82 |   device = "png",
83 |   vr.vars= c("average_montly_hours", "time_spend_company"),
84 |   vr.type = "partial")
85 | }
86 | }
87 | \author{
88 | Przemysław Biecek, Magda Tatarynowicz, Kamil Romaszko, Mateusz Urbański
89 | }
90 | 


--------------------------------------------------------------------------------
/modelDown.Rproj:
--------------------------------------------------------------------------------
 1 | Version: 1.0
 2 | 
 3 | RestoreWorkspace: No
 4 | SaveWorkspace: No
 5 | AlwaysSaveHistory: Default
 6 | 
 7 | EnableCodeIndexing: Yes
 8 | UseSpacesForTab: Yes
 9 | NumSpacesForTab: 2
10 | Encoding: UTF-8
11 | 
12 | RnwWeave: Sweave
13 | LaTeX: pdfLaTeX
14 | 
15 | AutoAppendNewline: Yes
16 | StripTrailingWhitespace: Yes
17 | 
18 | BuildType: Package
19 | PackageUseDevtools: Yes
20 | PackageInstallArgs: --no-multiarch --with-keep.source
21 | PackageRoxygenize: rd,collate,namespace
22 | 


--------------------------------------------------------------------------------
/paper.bib:
--------------------------------------------------------------------------------
 1 | @Article{DALEX,
 2 |   title = {DALEX: Explainers for Complex Predictive Models in {R}},
 3 |   author = {Przemyslaw Biecek},
 4 |   archivePrefix = "arXiv",
 5 |   journal = {Journal of Machine Learning Research},
 6 |   year = {2018},
 7 |   volume = {19},
 8 |   pages = {1-5},
 9 |   number = {84},
10 |   url = {http://jmlr.org/papers/v19/18-416.html},
11 | }
12 | 
13 | @Book{PM_VEE,
14 |   author = {Przemyslaw Biecek and Tomasz Burzykowski},
15 |   title = {Predictive Models: Visual Exploration, Explanation and Debugging},
16 |   year = 2019,
17 |   url = {https://pbiecek.github.io/PM_VEE/},
18 |   urldate = {2019-04-14}
19 | }
20 | 
21 | @Article{Archivist,
22 |   title = {{archivist}: An {R} Package for Managing, Recording and Restoring Data Analysis Results},
23 |   author = {Przemyslaw Biecek and Marcin Kosinski},
24 |   archivePrefix = "arXiv",
25 |   journal = {Journal of Statistical Software},
26 |   year = {2017},
27 |   volume = {82},
28 |   number = {11},
29 |   pages = {1--28},
30 |   doi = {10.18637/jss.v082.i11},
31 | }
32 | 
33 | @Manual{Drifter,
34 |   title = {{drifter}: Concept Drift and Concept Shift Detection for Predictive Models},
35 |   author = {Przemyslaw Biecek},
36 |   year = {2019},
37 |   note = {R package version 0.1},
38 |   url = {https://ModelOriented.github.io/drifter/},
39 | }
40 | 
41 | @Manual{Auditor,
42 |   title = {{auditor}: Model Audit - Verification, Validation, and Error Analysis},
43 |   author = {Alicja Gosiewska and Przemyslaw Biecek},
44 |   year = {2018},
45 |   note = {R package version 0.3.1},
46 |   url = {https://CRAN.R-project.org/package=auditor},
47 | }
48 | 
49 | @Manual{R,
50 |   title = {R: A Language and Environment for Statistical Computing},
51 |   author = {{R Core Team}},
52 |   organization = {R Foundation for Statistical Computing},
53 |   address = {Vienna, Austria},
54 |   year = {2018},
55 |   url = {https://www.R-project.org/},
56 | }
57 | 
58 | @online{DrWhy,
59 |   author = {MI2DataLab @ Warsaw University of Technology},
60 |   title = {DrWhy: collection of tools for Explainable {AI (XAI)}},
61 |   year = 2018,
62 |   url = {http://www.drwhy.ai/},
63 |   urldate = {2019-04-14}
64 | }
65 | 
66 | @Manual{pkgdown,
67 |   title = {{pkgdown}: Make Static {HTML} Documentation for a Package},
68 |   author = {Hadley Wickham and Jay Hesselberth},
69 |   year = {2018},
70 |   note = {R package version 1.3.0},
71 |   url = {https://CRAN.R-project.org/package=pkgdown},
72 | }
73 |  
74 | @Manual{whisker,
75 |   title = {{whisker}: mustache for {R}, logicless templating},
76 |   author = {Edwin {de Jonge}},
77 |   year = {2013},
78 |   note = {R package version 0.3-2},
79 |   url = {https://CRAN.R-project.org/package=whisker},
80 | }
81 | 
82 | @Book{ggplot2,
83 |   author = {Hadley Wickham},
84 |   title = {ggplot2: Elegant Graphics for Data Analysis},
85 |   publisher = {Springer-Verlag New York},
86 |   year = {2016},
87 |   isbn = {978-3-319-24277-4},
88 |   url = {http://ggplot2.org},
89 | }
90 | 
91 | @Manual{svg,
92 |   title = {{svglite}: An {'SVG'} Graphics Device},
93 |   author = {Hadley Wickham and Lionel Henry and T Jake Luciani and Matthieu Decorde and Vaudor Lise},
94 |   year = {2017},
95 |   note = {R package version 1.2.1},
96 |   url = {https://CRAN.R-project.org/package=svglite},
97 | }
98 | 


--------------------------------------------------------------------------------
/paper.md:
--------------------------------------------------------------------------------
  1 | ---
  2 | title: 'modelDown: automated website generator with interpretable documentation for predictive machine learning models'
  3 | tags:
  4 |   - R
  5 |   - explainable artificial intelligence
  6 |   - interpretable machine learning
  7 |   - predictive models
  8 |   - model visualization
  9 |   - automated data analysis
 10 | authors:
 11 |   - name: Kamil Romaszko
 12 |     affiliation: 1
 13 |   - name: Magda Tatarynowicz
 14 |     affiliation: 1
 15 |   - name: Mateusz Urbański
 16 |     affiliation: 1
 17 |   - name: Przemysław Biecek
 18 |     orcid: 0000-0001-8423-1823
 19 |     affiliation: 1
 20 | affiliations:
 21 |  - name: Faculty of Mathematics and Information Science, Warsaw University of Technology
 22 |    index: 1
 23 | date: 14 August 2019
 24 | bibliography: paper.bib
 25 | ---
 26 | 
 27 | # Introduction
 28 | 
 29 | We live in times where machine learning is used almost everywhere. With the large number of accessible tools available, generation of predictive model for your own benefit is fairly straightforward. 
 30 | 
 31 | The models may be simple to create and use, but most of the time they are not simple to understand. Larger models may contain thousands or millions of parameters. It can be challenging to see how the model works and how the inputs impact the obtained results. Numerous tools for explainable Artificial Intelligence can help, but it may be cumbersome and repetitive to use different tools every time you test the model. 
 32 | 
 33 | In this paper, we introduce a package modelDown for the R language [@R] that can help to eliminate these issues. modelDown automatically creates an easy to understand HTML website with model interpretations. You can automatically analyze, explore, and debug your predictive models in many ways with just a single command.
 34 | 
 35 | The modelDown package is designed in a similar way to the pkgdown [@pkgdown] library. Both packages use templates to generate a static HTML website with detailed information. In the case of pkgdown, the information is about a selected package, while in the case of modelDown, it is about predictive models.
 36 | 
 37 | The modelDown package builds on the set of tools for model exploration implemented in the DALEX [@DALEX] package. By creating explainer objects, DALEX provides a uniform interface for model agnostic exploration of models. Other R packages compatible with DALEX are also used to provide data, tables, and charts available on the website.
 38 | 
 39 | # Example code
 40 | 
 41 | A basic example of using the package is as straightforward as below.
 42 | 
 43 | ```
 44 | require("ranger")
 45 | require("breakDown")
 46 | require("DALEX")
 47 | 
 48 | devtools::install_github("MI2DataLab/modelDown")
 49 | 
 50 | # ranger
 51 | HR_ranger_model <- ranger(as.factor(left) ~ .,
 52 |                           data = HR_data, num.trees = 500,
 53 |                           classification = TRUE, probability = TRUE)
 54 | 
 55 | # glm
 56 | HR_data1 <- HR_data[1:4000,]
 57 | HR_data2 <- HR_data[4000:nrow(HR_data),]
 58 | HR_glm_model1 <- glm(left~., HR_data1, family = "binomial")
 59 | HR_glm_model2 <- glm(left~., HR_data2, family = "binomial")
 60 | 
 61 | # generating explainers
 62 | explainer_ranger <- explain(HR_ranger_model,
 63 |                             data = HR_data, y = HR_data$left,
 64 |                             function(model, data) {
 65 |                               return(predict(model, data)$prediction[,2])
 66 |                             }, na.rm=TRUE)
 67 | explainer_glm1 <- explain(HR_glm_model1, data=HR_data1, y = HR_data1$left)
 68 | explainer_glm2 <- explain(HR_glm_model2, data=HR_data2, y = HR_data2$left)
 69 | 
 70 | modelDown::modelDown(explainer_ranger, list(explainer_glm1, explainer_glm2))
 71 | ```
 72 | 
 73 | # Available features
 74 | 
 75 | The modelDown package is highly modular. The primary outcome is a website with a set of tabs. Each tab explains a particular aspect of the model. Features are available across the tabs. By default, all the tabs are generated, and each of them can be parameterized. 
 76 | 
 77 | The 'Auditor' tab presents different ways of validating predictive models by analysis of residuals. All plots are generated by the auditor package [@Auditor]. The plots may seem a bit complicated at first sight, so each plot has links with helpful explanations. 
 78 | 
 79 | The 'Model performance' tab uses functions from DALEX package [@DALEX]. It shows contrastive comparisons of model predictive power by analyzing residuals. Read more about this type of model diagnostic in [@PM_VEE].
 80 | 
 81 | The 'Model performance' tab lets you compare different models. This applies for all tabs created with modelDown. This allows users not only to validate one model, but to pick the best model among many. Below we can see different distributions of residuals for the models.
 82 | 
 83 | -![Example model performance tab.](paper_images/model_performance.png)
 84 | 
 85 | The 'Feature importance' tab also uses functions from DALEX package. It shows how much impact each variable has when it comes to generating predictions. You can compare these model agnostic feature importance scores against domain knowledge in order to see if the model works in the expected way.
 86 | 
 87 | -![Example variable importance tab.](paper_images/variable_importance.png)
 88 | 
 89 | All examples presented in this paper are based on models generated with the Titanic dataset. In this example, three models that are considered picked gender as the most important variable. Only the k-Nearest Neighbors algorithm picked different features. This may suggest that this model does not give the best results for this data.
 90 | 
 91 | The 'Feature response' tab also uses functions from DALEX package. The main goal of this table is to present the link between average model response and the feature of interest. With the use of Partial Dependency Plots or Factor Merger Plots one can read how model predictions change with changing values of a selected single variable.
 92 | 
 93 | The last tab, 'drifter', uses functions from an R package with the same name [@Drifter]. You can test if there is concept drift or data drift in the data. You can also see how changes in the data distribution impact the model of interest.
 94 | 
 95 | All of this can be generated using one simple command, which is highly customizable. You can select different plot formats - either a raster png or vectorized [@svg], change each tab's individual settings and control visual aspects.  
 96 | 
 97 | To boost reproducibility of models, all results from model exploration and explanation are saved in binary form with the archivist [@Archivist] package. You can always look at them later, share models or explainers with collaborators, validate session info recorded when explainers were created, or reproduce selected explainers.
 98 | 
 99 | # Summary
100 | 
101 | The package uses whisker html templates [@whisker]. Also, DALEX itself uses the extensible grammar of data visualization implemented in the ggplot2 package [@ggplot2]. Because of that, as new packages compatible with DALEX explainers are developed, modelDown can be extended to use them. modelDown, with its simple usage and wide variety of information on the models, may be a powerful tool that helps you improve your model predictive power.
102 | 
103 | An example website generated with the modelDown package is at https://github.com/MI2DataLab/modelDown_example. It also contains a reproducible R code for model training, wrapping and transforming into an HTML website.
104 | 
105 | The modelDown is a part of DrWhy.AI [@DrWhy] collection of tools for automated training, exploration and maintenance for predictive models.
106 | 
107 | # Acknowledgments
108 | 
109 | Work on this package was financially supported by the NCN Opus grant<br/>  
110 | 2017/27/B/ST6/01307 at Warsaw University of Technology, Faculty of Mathematics and Information Science.
111 | 
112 | # References
113 | 
114 | 
115 | 


--------------------------------------------------------------------------------
/paper_images/model_performance.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/paper_images/model_performance.png


--------------------------------------------------------------------------------
/paper_images/variable_importance.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/paper_images/variable_importance.png


--------------------------------------------------------------------------------
/pkgdown/_pkgdown.yml:
--------------------------------------------------------------------------------
1 | template:
2 |   package: DrWhyTemplate
3 |   default_assets: false
4 | 


--------------------------------------------------------------------------------
/pkgdown/favicon/apple-touch-icon-120x120.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/pkgdown/favicon/apple-touch-icon-120x120.png


--------------------------------------------------------------------------------
/pkgdown/favicon/apple-touch-icon-152x152.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/pkgdown/favicon/apple-touch-icon-152x152.png


--------------------------------------------------------------------------------
/pkgdown/favicon/apple-touch-icon-180x180.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/pkgdown/favicon/apple-touch-icon-180x180.png


--------------------------------------------------------------------------------
/pkgdown/favicon/apple-touch-icon-60x60.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/pkgdown/favicon/apple-touch-icon-60x60.png


--------------------------------------------------------------------------------
/pkgdown/favicon/apple-touch-icon-76x76.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/pkgdown/favicon/apple-touch-icon-76x76.png


--------------------------------------------------------------------------------
/pkgdown/favicon/apple-touch-icon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/pkgdown/favicon/apple-touch-icon.png


--------------------------------------------------------------------------------
/pkgdown/favicon/favicon-16x16.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/pkgdown/favicon/favicon-16x16.png


--------------------------------------------------------------------------------
/pkgdown/favicon/favicon-32x32.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/pkgdown/favicon/favicon-32x32.png


--------------------------------------------------------------------------------
/pkgdown/favicon/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ModelOriented/modelDown/c1f230791c2c6412e7239b98e11db5bba05e8248/pkgdown/favicon/favicon.ico


--------------------------------------------------------------------------------
/scripts/regenerate_pkgdown_docs.R:
--------------------------------------------------------------------------------
1 | install.packages("pkgdown")
2 | install.packages("testthat")
3 | devtools::install_github("ModelOriented/DrWhyTemplate")
4 | 
5 | pkgdown::build_site()
6 | 


--------------------------------------------------------------------------------
/tests/testthat.R:
--------------------------------------------------------------------------------
1 | library(testthat)
2 | library(DALEX)
3 | library(devtools)
4 | library(modelDown)
5 | 
6 | test_check("modelDown")
7 | 


--------------------------------------------------------------------------------
/tests/testthat/test_modelDown.R:
--------------------------------------------------------------------------------
 1 | context("Check modelDown() function")
 2 | 
 3 | test_that("Default arguments", {
 4 |   expect_true({
 5 |     require("ranger")
 6 |     require("breakDown")
 7 |     require("DALEX")
 8 |     HR_data_subset = head(breakDown::HR_data, 3000)
 9 |     HR_data_subset2 = head(breakDown::HR_data, 4000)
10 | 
11 |     # ranger
12 |     HR_ranger_model <- ranger(
13 |       as.factor(left) ~ .,
14 |       data = HR_data_subset,
15 |       num.trees = 500,
16 |       classification = TRUE,
17 |       probability = TRUE
18 |     )
19 |     explainer_ranger <- explain(HR_ranger_model,
20 |                                 data = HR_data_subset, y = HR_data_subset$left, function(model, data) {
21 |                                   return(predict(model, data)$prediction[, 2])
22 |                                 }, na.rm = TRUE)
23 | 
24 |     # glm
25 |     HR_glm_model1 <- glm(left ~ ., HR_data_subset, family = "binomial")
26 |     HR_glm_model2 <- glm(left ~ ., HR_data_subset2, family = "binomial")
27 |     explainer_glm1 <-
28 |       explain(HR_glm_model1, data = HR_data_subset, y = HR_data_subset$left)
29 |     explainer_glm2 <-
30 |       explain(HR_glm_model2, data = HR_data_subset2, y = HR_data_subset2$left)
31 | 
32 |     modelDown::modelDown(explainer_ranger, list(explainer_glm1, explainer_glm2),
33 |                          output_folder = "modelDown_tmp")
34 | 
35 |     TRUE
36 |   })
37 | })
38 | 
39 | 
40 | test_that("Validation for character variable in explainer dataset", {
41 |   expect_error({
42 |     require("DALEX")
43 | 
44 |     titanic_char <- titanic
45 |     titanic_char$country_character <- as.character(titanic$country)
46 | 
47 |     titanic_glm_model <- glm(survived == "yes" ~ ., titanic_char, family = "binomial")
48 | 
49 |     explainer_glm <-
50 |       explain(titanic_glm_model, data = titanic_char, y = titanic_char$survived)
51 |     modelDown::modelDown(explainer_glm)
52 |   })
53 | })
54 | 


--------------------------------------------------------------------------------
/tests/testthat/test_parseExplainers.R:
--------------------------------------------------------------------------------
 1 | context("Check parseExplainers() function")
 2 | library(useful)
 3 | 
 4 | createFakeExplainer <- function(name) {
 5 |   object <- list()
 6 |   object$name <- name
 7 |   class(object) <- "explainer"
 8 |   return(object)
 9 | }
10 | 
11 | old1 <- createFakeExplainer("old1")
12 | new1 <- createFakeExplainer("new1")
13 | old2 <- createFakeExplainer("old2")
14 | new2 <- createFakeExplainer("new2")
15 | old3 <- createFakeExplainer("old3")
16 | new3 <- createFakeExplainer("new3")
17 | 
18 | test_that("Default arguments", {
19 |   expect_true({
20 | 
21 |     explainers = list(old1, old2, old3)
22 |     res <- parseExplainers(explainers)
23 |     expected <- list(basic_explainers=list(old1, old2, old3), drifter_explainer_pairs=list())
24 | 
25 |     all(compare.list(res, expected))
26 |   })
27 | })
28 | 
29 | test_that("Mixed arguments", {
30 |   expect_true({
31 |     explainers = list(old1, list(old2, new2), old3)
32 |     res <- parseExplainers(explainers)
33 |     expected <- list(basic_explainers=list(old1, old2, old3), drifter_explainer_pairs=list(list(old2, new2)))
34 | 
35 |     all(compare.list(res, expected))
36 |   })
37 | })
38 | 
39 | test_that("To many explainers in vector", {
40 |   expect_error({
41 |     explainers = list(list(old2, new2, new3))
42 |     parseExplainers(explainers)
43 |   })
44 | })
45 | 


--------------------------------------------------------------------------------