├── README.md ├── benchmark ├── corrgroups60 │ ├── index.html │ ├── plot_corrgroups60_decision_tree_keep_absolute_impute__r2.pdf │ ├── plot_corrgroups60_decision_tree_keep_absolute_mask__r2.pdf │ ├── plot_corrgroups60_decision_tree_keep_absolute_resample__r2.pdf │ ├── plot_corrgroups60_decision_tree_keep_negative_impute.pdf │ ├── plot_corrgroups60_decision_tree_keep_negative_mask.pdf │ ├── plot_corrgroups60_decision_tree_keep_negative_resample.pdf │ ├── plot_corrgroups60_decision_tree_keep_positive_impute.pdf │ ├── plot_corrgroups60_decision_tree_keep_positive_mask.pdf │ ├── plot_corrgroups60_decision_tree_keep_positive_resample.pdf │ ├── plot_corrgroups60_decision_tree_remove_absolute_impute__r2.pdf │ ├── plot_corrgroups60_decision_tree_remove_absolute_mask__r2.pdf │ ├── plot_corrgroups60_decision_tree_remove_absolute_resample__r2.pdf │ ├── plot_corrgroups60_decision_tree_remove_negative_impute.pdf │ ├── plot_corrgroups60_decision_tree_remove_negative_mask.pdf │ ├── plot_corrgroups60_decision_tree_remove_negative_resample.pdf │ ├── plot_corrgroups60_decision_tree_remove_positive_impute.pdf │ ├── plot_corrgroups60_decision_tree_remove_positive_mask.pdf │ ├── plot_corrgroups60_decision_tree_remove_positive_resample.pdf │ ├── plot_corrgroups60_gbm_keep_absolute_impute__r2.pdf │ ├── plot_corrgroups60_gbm_keep_absolute_mask__r2.pdf │ ├── plot_corrgroups60_gbm_keep_absolute_resample__r2.pdf │ ├── plot_corrgroups60_gbm_keep_negative_impute.pdf │ ├── plot_corrgroups60_gbm_keep_negative_mask.pdf │ ├── plot_corrgroups60_gbm_keep_negative_resample.pdf │ ├── plot_corrgroups60_gbm_keep_positive_impute.pdf │ ├── plot_corrgroups60_gbm_keep_positive_mask.pdf │ ├── plot_corrgroups60_gbm_keep_positive_resample.pdf │ ├── plot_corrgroups60_gbm_remove_absolute_impute__r2.pdf │ ├── plot_corrgroups60_gbm_remove_absolute_mask__r2.pdf │ ├── plot_corrgroups60_gbm_remove_absolute_resample__r2.pdf │ ├── plot_corrgroups60_gbm_remove_negative_impute.pdf │ ├── plot_corrgroups60_gbm_remove_negative_mask.pdf │ ├── plot_corrgroups60_gbm_remove_negative_resample.pdf │ ├── plot_corrgroups60_gbm_remove_positive_impute.pdf │ ├── plot_corrgroups60_gbm_remove_positive_mask.pdf │ ├── plot_corrgroups60_gbm_remove_positive_resample.pdf │ ├── plot_corrgroups60_random_forest_keep_absolute_impute__r2.pdf │ ├── plot_corrgroups60_random_forest_keep_absolute_mask__r2.pdf │ ├── plot_corrgroups60_random_forest_keep_absolute_resample__r2.pdf │ ├── plot_corrgroups60_random_forest_keep_negative_impute.pdf │ ├── plot_corrgroups60_random_forest_keep_negative_mask.pdf │ ├── plot_corrgroups60_random_forest_keep_negative_resample.pdf │ ├── plot_corrgroups60_random_forest_keep_positive_impute.pdf │ ├── plot_corrgroups60_random_forest_keep_positive_mask.pdf │ ├── plot_corrgroups60_random_forest_keep_positive_resample.pdf │ ├── plot_corrgroups60_random_forest_remove_absolute_impute__r2.pdf │ ├── plot_corrgroups60_random_forest_remove_absolute_mask__r2.pdf │ ├── plot_corrgroups60_random_forest_remove_absolute_resample__r2.pdf │ ├── plot_corrgroups60_random_forest_remove_negative_impute.pdf │ ├── plot_corrgroups60_random_forest_remove_negative_mask.pdf │ ├── plot_corrgroups60_random_forest_remove_negative_resample.pdf │ ├── plot_corrgroups60_random_forest_remove_positive_impute.pdf │ ├── plot_corrgroups60_random_forest_remove_positive_mask.pdf │ └── plot_corrgroups60_random_forest_remove_positive_resample.pdf ├── cric │ ├── index.html │ ├── plot_cric_decision_tree_keep_absolute_impute__roc_auc.pdf │ ├── plot_cric_decision_tree_keep_absolute_mask__roc_auc.pdf │ ├── plot_cric_decision_tree_keep_absolute_resample__roc_auc.pdf │ ├── plot_cric_decision_tree_keep_negative_impute.pdf │ ├── plot_cric_decision_tree_keep_negative_mask.pdf │ ├── plot_cric_decision_tree_keep_negative_resample.pdf │ ├── plot_cric_decision_tree_keep_positive_impute.pdf │ ├── plot_cric_decision_tree_keep_positive_mask.pdf │ ├── plot_cric_decision_tree_keep_positive_resample.pdf │ ├── plot_cric_decision_tree_remove_absolute_impute__roc_auc.pdf │ ├── plot_cric_decision_tree_remove_absolute_mask__roc_auc.pdf │ ├── plot_cric_decision_tree_remove_absolute_resample__roc_auc.pdf │ ├── plot_cric_decision_tree_remove_negative_impute.pdf │ ├── plot_cric_decision_tree_remove_negative_mask.pdf │ ├── plot_cric_decision_tree_remove_negative_resample.pdf │ ├── plot_cric_decision_tree_remove_positive_impute.pdf │ ├── plot_cric_decision_tree_remove_positive_mask.pdf │ ├── plot_cric_decision_tree_remove_positive_resample.pdf │ ├── plot_cric_gbm_keep_absolute_impute__roc_auc.pdf │ ├── plot_cric_gbm_keep_absolute_mask__roc_auc.pdf │ ├── plot_cric_gbm_keep_absolute_resample__roc_auc.pdf │ ├── plot_cric_gbm_keep_negative_impute.pdf │ ├── plot_cric_gbm_keep_negative_mask.pdf │ ├── plot_cric_gbm_keep_negative_resample.pdf │ ├── plot_cric_gbm_keep_positive_impute.pdf │ ├── plot_cric_gbm_keep_positive_mask.pdf │ ├── plot_cric_gbm_keep_positive_resample.pdf │ ├── plot_cric_gbm_remove_absolute_impute__roc_auc.pdf │ ├── plot_cric_gbm_remove_absolute_mask__roc_auc.pdf │ ├── plot_cric_gbm_remove_absolute_resample__roc_auc.pdf │ ├── plot_cric_gbm_remove_negative_impute.pdf │ ├── plot_cric_gbm_remove_negative_mask.pdf │ ├── plot_cric_gbm_remove_negative_resample.pdf │ ├── plot_cric_gbm_remove_positive_impute.pdf │ ├── plot_cric_gbm_remove_positive_mask.pdf │ ├── plot_cric_gbm_remove_positive_resample.pdf │ ├── plot_cric_random_forest_keep_absolute_impute__roc_auc.pdf │ ├── plot_cric_random_forest_keep_absolute_mask__roc_auc.pdf │ ├── plot_cric_random_forest_keep_absolute_resample__roc_auc.pdf │ ├── plot_cric_random_forest_keep_negative_impute.pdf │ ├── plot_cric_random_forest_keep_negative_mask.pdf │ ├── plot_cric_random_forest_keep_negative_resample.pdf │ ├── plot_cric_random_forest_keep_positive_impute.pdf │ ├── plot_cric_random_forest_keep_positive_mask.pdf │ ├── plot_cric_random_forest_keep_positive_resample.pdf │ ├── plot_cric_random_forest_remove_absolute_impute__roc_auc.pdf │ ├── plot_cric_random_forest_remove_absolute_mask__roc_auc.pdf │ ├── plot_cric_random_forest_remove_absolute_resample__roc_auc.pdf │ ├── plot_cric_random_forest_remove_negative_impute.pdf │ ├── plot_cric_random_forest_remove_negative_mask.pdf │ ├── plot_cric_random_forest_remove_negative_resample.pdf │ ├── plot_cric_random_forest_remove_positive_impute.pdf │ ├── plot_cric_random_forest_remove_positive_mask.pdf │ └── plot_cric_random_forest_remove_positive_resample.pdf ├── human │ ├── index.html │ ├── plot_human_decision_tree_human_and_00.pdf │ ├── plot_human_decision_tree_human_and_01.pdf │ ├── plot_human_decision_tree_human_and_11.pdf │ ├── plot_human_decision_tree_human_or_00.pdf │ ├── plot_human_decision_tree_human_or_01.pdf │ ├── plot_human_decision_tree_human_or_11.pdf │ ├── plot_human_decision_tree_human_sum_00.pdf │ ├── plot_human_decision_tree_human_sum_01.pdf │ ├── plot_human_decision_tree_human_sum_11.pdf │ ├── plot_human_decision_tree_human_xor_00.pdf │ ├── plot_human_decision_tree_human_xor_01.pdf │ └── plot_human_decision_tree_human_xor_11.pdf └── independentlinear60 │ ├── index.html │ ├── plot_independentlinear60_decision_tree_keep_absolute_impute__r2.pdf │ ├── plot_independentlinear60_decision_tree_keep_absolute_mask__r2.pdf │ ├── plot_independentlinear60_decision_tree_keep_absolute_resample__r2.pdf │ ├── plot_independentlinear60_decision_tree_keep_negative_impute.pdf │ ├── plot_independentlinear60_decision_tree_keep_negative_mask.pdf │ ├── plot_independentlinear60_decision_tree_keep_negative_resample.pdf │ ├── plot_independentlinear60_decision_tree_keep_positive_impute.pdf │ ├── plot_independentlinear60_decision_tree_keep_positive_mask.pdf │ ├── plot_independentlinear60_decision_tree_keep_positive_resample.pdf │ ├── plot_independentlinear60_decision_tree_remove_absolute_impute__r2.pdf │ ├── plot_independentlinear60_decision_tree_remove_absolute_mask__r2.pdf │ ├── plot_independentlinear60_decision_tree_remove_absolute_resample__r2.pdf │ ├── plot_independentlinear60_decision_tree_remove_negative_impute.pdf │ ├── plot_independentlinear60_decision_tree_remove_negative_mask.pdf │ ├── plot_independentlinear60_decision_tree_remove_negative_resample.pdf │ ├── plot_independentlinear60_decision_tree_remove_positive_impute.pdf │ ├── plot_independentlinear60_decision_tree_remove_positive_mask.pdf │ ├── plot_independentlinear60_decision_tree_remove_positive_resample.pdf │ ├── plot_independentlinear60_gbm_keep_absolute_impute__r2.pdf │ ├── plot_independentlinear60_gbm_keep_absolute_mask__r2.pdf │ ├── plot_independentlinear60_gbm_keep_absolute_resample__r2.pdf │ ├── plot_independentlinear60_gbm_keep_negative_impute.pdf │ ├── plot_independentlinear60_gbm_keep_negative_mask.pdf │ ├── plot_independentlinear60_gbm_keep_negative_resample.pdf │ ├── plot_independentlinear60_gbm_keep_positive_impute.pdf │ ├── plot_independentlinear60_gbm_keep_positive_mask.pdf │ ├── plot_independentlinear60_gbm_keep_positive_resample.pdf │ ├── plot_independentlinear60_gbm_remove_absolute_impute__r2.pdf │ ├── plot_independentlinear60_gbm_remove_absolute_mask__r2.pdf │ ├── plot_independentlinear60_gbm_remove_absolute_resample__r2.pdf │ ├── plot_independentlinear60_gbm_remove_negative_impute.pdf │ ├── plot_independentlinear60_gbm_remove_negative_mask.pdf │ ├── plot_independentlinear60_gbm_remove_negative_resample.pdf │ ├── plot_independentlinear60_gbm_remove_positive_impute.pdf │ ├── plot_independentlinear60_gbm_remove_positive_mask.pdf │ ├── plot_independentlinear60_gbm_remove_positive_resample.pdf │ ├── plot_independentlinear60_random_forest_keep_absolute_impute__r2.pdf │ ├── plot_independentlinear60_random_forest_keep_absolute_mask__r2.pdf │ ├── plot_independentlinear60_random_forest_keep_absolute_resample__r2.pdf │ ├── plot_independentlinear60_random_forest_keep_negative_impute.pdf │ ├── plot_independentlinear60_random_forest_keep_negative_mask.pdf │ ├── plot_independentlinear60_random_forest_keep_negative_resample.pdf │ ├── plot_independentlinear60_random_forest_keep_positive_impute.pdf │ ├── plot_independentlinear60_random_forest_keep_positive_mask.pdf │ ├── plot_independentlinear60_random_forest_keep_positive_resample.pdf │ ├── plot_independentlinear60_random_forest_remove_absolute_impute__r2.pdf │ ├── plot_independentlinear60_random_forest_remove_absolute_mask__r2.pdf │ ├── plot_independentlinear60_random_forest_remove_absolute_resample__r2.pdf │ ├── plot_independentlinear60_random_forest_remove_negative_impute.pdf │ ├── plot_independentlinear60_random_forest_remove_negative_mask.pdf │ ├── plot_independentlinear60_random_forest_remove_negative_resample.pdf │ ├── plot_independentlinear60_random_forest_remove_positive_impute.pdf │ ├── plot_independentlinear60_random_forest_remove_positive_mask.pdf │ └── plot_independentlinear60_random_forest_remove_positive_resample.pdf ├── images └── overview_header.png ├── notebooks ├── Saabas Inconsistencies.ipynb ├── kidney │ ├── CRIC Analysis.ipynb │ └── convergence_plots │ │ ├── .ipynb_checkpoints │ │ ├── Generate and Save Deviations-checkpoint.ipynb │ │ └── Plots-checkpoint.ipynb │ │ ├── Generate and Save Deviations.ipynb │ │ ├── Plots.ipynb │ │ ├── kernel_expl_abs_dev_single_ref_truth_max_dict.p │ │ ├── kernel_expl_abs_dev_single_ref_truth_mean_dict.p │ │ ├── max_kidney_agnostic_convergence.pdf │ │ ├── mean_kidney_agnostic_convergence.pdf │ │ ├── sample_expl_abs_dev_single_ref_truth_max_dict.p │ │ └── sample_expl_abs_dev_single_ref_truth_mean_dict.p ├── mortality │ ├── NHANES I Analysis.ipynb │ ├── NHANES Nonlinearity.ipynb │ ├── data │ │ ├── DU4111.txt │ │ ├── DU4171.txt │ │ ├── DU4233.txt │ │ ├── DU4233_F.sas │ │ ├── DU4800.txt │ │ ├── N92vitl.txt │ │ ├── X_orig.csv │ │ └── y_orig.npy │ └── loadnhanes.py ├── simulation │ ├── .ipynb_checkpoints │ │ ├── Generate and Save Deviations-checkpoint.ipynb │ │ └── Plot Runtime and Convergence-checkpoint.ipynb │ ├── Generate and Save Deviations.ipynb │ ├── Plot Runtime and Convergence.ipynb │ ├── Saabas Inconsistencies.ipynb │ └── scratch │ │ ├── ime_m_lst.npy │ │ ├── ime_std_lst.npy │ │ ├── kernel_shap_m_lst.npy │ │ └── kernel_shap_std_lst.npy └── timeseries │ ├── Explain LSTM->XGB.ipynb │ ├── Explain XGB.ipynb │ ├── Run LSTM->XGB.ipynb │ ├── Run LSTM.ipynb │ ├── Run XGB.ipynb │ └── tbi_downstream_prediction.py └── user_studies └── sickness_scores ├── .ipynb_checkpoints └── Analyse Results-checkpoint.ipynb ├── .psiturk_history ├── Analyse Results.ipynb ├── Debrief.jsx ├── Instructions.jsx ├── Main.jsx ├── Questions.jsx ├── README.md ├── config.txt ├── index.jsx ├── package.json ├── static ├── css │ ├── bootstrap.min.css │ └── style.css ├── favicon.ico ├── fonts │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.svg │ ├── glyphicons-halflings-regular.ttf │ └── glyphicons-halflings-regular.woff ├── images │ ├── Example1.png │ ├── Example1_nobar.png │ ├── Example2.png │ ├── Example2_nobar.png │ └── university.png ├── js │ ├── bundle.js │ ├── task.js │ └── utils.js └── lib │ ├── backbone-min.js │ ├── bootstrap.min.js │ ├── d3.v3.min.js │ ├── jquery-min.js │ └── underscore-min.js ├── templates ├── ad.html ├── closepopup.html ├── complete.html ├── consent.html ├── default.html ├── error.html ├── exp.html ├── thanks-mturksubmit.html └── thanks.html └── webpack.config.js /README.md: -------------------------------------------------------------------------------- 1 | # TreeExplainer Paper Repository 2 | 3 |

4 | 5 |

6 | 7 | --- 8 | 9 | This repository contains notebook code and data for the paper: *Explainable AI for Trees: From Local Explanations to Global Understanding* 10 | 11 | This paper presents the first exact game theoretic solution for explaining predictions from tree ensemble models, the most widely used non-linear machine learning models; this solution is the only one that guarantees desirable properties and so enables new insights into individual predictions and global behavior of the model. 12 | 13 | **Ways to explore the results and code behind the paper:** 14 | - Use TreeExplainer to explain your own tree-based models with the popular Python [shap package](https://github.com/slundberg/shap). 15 | * System requirements: Any recent Mac, Windows, or Linux OS. See https://anaconda.org/conda-forge/shap/files for a list of systems with precompiled binaries the codes is unit tested on. 16 | * Installation: See https://github.com/slundberg/shap (install time is typically < 1 min) 17 | * Demo: See the TreeExplainer demos at https://github.com/slundberg/shap, the demo on the front page runs in a few seconds on any modern computer. 18 | 19 | - Check out the [local explanation benchmark](https://suinleelab.github.io/treeexplainer-study/benchmark/cric/index.html) results. 20 | 21 | - Download the [NHANES I data](https://github.com/suinleelab/treeexplainer-study/tree/master/notebooks/mortality) (the chronic kidney disease and hospital datasets require approval for access) 22 | 23 | - Browse the [released notebooks](https://github.com/suinleelab/treeexplainer-study/tree/master/notebooks) (still under construction). 24 | -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_keep_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_decision_tree_remove_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_keep_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_keep_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_keep_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_keep_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_keep_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_keep_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_keep_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_keep_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_keep_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_keep_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_keep_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_keep_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_keep_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_keep_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_keep_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_keep_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_keep_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_keep_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_remove_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_remove_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_remove_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_remove_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_remove_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_remove_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_remove_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_remove_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_remove_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_remove_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_remove_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_remove_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_remove_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_remove_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_remove_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_remove_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_gbm_remove_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_gbm_remove_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_keep_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/corrgroups60/plot_corrgroups60_random_forest_remove_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_keep_absolute_impute__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_keep_absolute_impute__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_keep_absolute_mask__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_keep_absolute_mask__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_keep_absolute_resample__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_keep_absolute_resample__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_keep_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_keep_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_keep_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_keep_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_keep_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_keep_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_keep_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_keep_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_keep_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_keep_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_keep_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_keep_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_remove_absolute_impute__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_remove_absolute_impute__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_remove_absolute_mask__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_remove_absolute_mask__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_remove_absolute_resample__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_remove_absolute_resample__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_remove_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_remove_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_remove_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_remove_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_remove_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_remove_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_remove_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_remove_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_remove_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_remove_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_decision_tree_remove_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_decision_tree_remove_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_keep_absolute_impute__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_keep_absolute_impute__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_keep_absolute_mask__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_keep_absolute_mask__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_keep_absolute_resample__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_keep_absolute_resample__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_keep_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_keep_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_keep_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_keep_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_keep_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_keep_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_keep_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_keep_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_keep_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_keep_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_keep_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_keep_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_remove_absolute_impute__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_remove_absolute_impute__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_remove_absolute_mask__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_remove_absolute_mask__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_remove_absolute_resample__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_remove_absolute_resample__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_remove_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_remove_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_remove_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_remove_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_remove_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_remove_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_remove_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_remove_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_remove_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_remove_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_gbm_remove_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_gbm_remove_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_keep_absolute_impute__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_keep_absolute_impute__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_keep_absolute_mask__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_keep_absolute_mask__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_keep_absolute_resample__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_keep_absolute_resample__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_keep_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_keep_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_keep_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_keep_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_keep_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_keep_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_keep_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_keep_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_keep_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_keep_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_keep_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_keep_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_remove_absolute_impute__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_remove_absolute_impute__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_remove_absolute_mask__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_remove_absolute_mask__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_remove_absolute_resample__roc_auc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_remove_absolute_resample__roc_auc.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_remove_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_remove_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_remove_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_remove_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_remove_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_remove_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_remove_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_remove_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_remove_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_remove_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/cric/plot_cric_random_forest_remove_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/cric/plot_cric_random_forest_remove_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_and_00.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_and_00.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_and_01.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_and_01.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_and_11.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_and_11.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_or_00.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_or_00.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_or_01.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_or_01.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_or_11.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_or_11.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_sum_00.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_sum_00.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_sum_01.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_sum_01.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_sum_11.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_sum_11.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_xor_00.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_xor_00.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_xor_01.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_xor_01.pdf -------------------------------------------------------------------------------- /benchmark/human/plot_human_decision_tree_human_xor_11.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/human/plot_human_decision_tree_human_xor_11.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_keep_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_decision_tree_remove_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_keep_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_keep_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_keep_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_keep_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_keep_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_keep_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_keep_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_keep_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_keep_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_keep_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_keep_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_keep_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_keep_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_keep_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_keep_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_keep_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_keep_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_keep_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_remove_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_remove_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_remove_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_remove_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_remove_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_remove_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_remove_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_remove_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_remove_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_remove_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_remove_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_remove_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_remove_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_remove_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_remove_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_remove_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_gbm_remove_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_gbm_remove_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_keep_positive_resample.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_absolute_impute__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_absolute_impute__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_absolute_mask__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_absolute_mask__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_absolute_resample__r2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_absolute_resample__r2.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_negative_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_negative_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_negative_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_negative_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_negative_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_negative_resample.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_positive_impute.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_positive_impute.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_positive_mask.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_positive_mask.pdf -------------------------------------------------------------------------------- /benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_positive_resample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/benchmark/independentlinear60/plot_independentlinear60_random_forest_remove_positive_resample.pdf -------------------------------------------------------------------------------- /images/overview_header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/images/overview_header.png -------------------------------------------------------------------------------- /notebooks/kidney/convergence_plots/kernel_expl_abs_dev_single_ref_truth_max_dict.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/notebooks/kidney/convergence_plots/kernel_expl_abs_dev_single_ref_truth_max_dict.p -------------------------------------------------------------------------------- /notebooks/kidney/convergence_plots/kernel_expl_abs_dev_single_ref_truth_mean_dict.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/notebooks/kidney/convergence_plots/kernel_expl_abs_dev_single_ref_truth_mean_dict.p -------------------------------------------------------------------------------- /notebooks/kidney/convergence_plots/max_kidney_agnostic_convergence.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/notebooks/kidney/convergence_plots/max_kidney_agnostic_convergence.pdf -------------------------------------------------------------------------------- /notebooks/kidney/convergence_plots/mean_kidney_agnostic_convergence.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/notebooks/kidney/convergence_plots/mean_kidney_agnostic_convergence.pdf -------------------------------------------------------------------------------- /notebooks/kidney/convergence_plots/sample_expl_abs_dev_single_ref_truth_max_dict.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/notebooks/kidney/convergence_plots/sample_expl_abs_dev_single_ref_truth_max_dict.p -------------------------------------------------------------------------------- /notebooks/kidney/convergence_plots/sample_expl_abs_dev_single_ref_truth_mean_dict.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/notebooks/kidney/convergence_plots/sample_expl_abs_dev_single_ref_truth_mean_dict.p -------------------------------------------------------------------------------- /notebooks/mortality/data/y_orig.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/notebooks/mortality/data/y_orig.npy -------------------------------------------------------------------------------- /notebooks/simulation/.ipynb_checkpoints/Generate and Save Deviations-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Generate and Save Sampling(IME)/Kernel Explainer convergence details for a simulated data set" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import xgboost\n", 17 | "import numpy as np\n", 18 | "import shap\n", 19 | "import time\n", 20 | "from tqdm import tqdm\n", 21 | "import matplotlib.pylab as pl\n", 22 | "from shap import KernelExplainer\n", 23 | "from shap import SamplingExplainer\n", 24 | "N = 1000\n", 25 | "Ms = [20,30,40,50,60,70,80,90]\n", 26 | "\n", 27 | "np.random.seed(3011)\n", 28 | "ime_std_lst =[]\n", 29 | "ime_m_lst = []\n", 30 | "kernel_shap_std_lst =[]\n", 31 | "kernel_shap_m_lst = []\n", 32 | "for l in range(0,30):\n", 33 | " X_full = np.random.randn(N, 100)\n", 34 | " y = np.random.randn(N)\n", 35 | " kernel_shap_nsamples = []\n", 36 | " ime_nsamples = []\n", 37 | " tree_shap_std = []\n", 38 | " kernel_shap_std = []\n", 39 | " kernel_shap_m = []\n", 40 | " ime_std = []\n", 41 | " ime_m = []\n", 42 | " for M in Ms:\n", 43 | " print(\"\\nM\", M)\n", 44 | " X = X_full[:,:M]\n", 45 | " model = xgboost.train({\"silent\":1}, xgboost.DMatrix(X, y), 1000)\n", 46 | "\n", 47 | " def f(x):\n", 48 | " return model.predict(xgboost.DMatrix(x))\n", 49 | "\n", 50 | " e = shap.TreeExplainer(model)\n", 51 | " e.shap_values(X)\n", 52 | " tree_shap_std.append(0)\n", 53 | "\n", 54 | " print(\"Kernel Explainer\") \n", 55 | " nsamples = 1000 * M\n", 56 | " print(nsamples)\n", 57 | " e = KernelExplainer(f, X.mean(0).reshape(1,M))\n", 58 | " out = np.vstack([e.shap_values(X[:1,:], silent=True, nsamples=nsamples) for i in range(10)])\n", 59 | " std_dev = out.std(0)[:-1].mean()\n", 60 | " mval = np.abs(out.mean(0))[:-1].mean()\n", 61 | " print(std_dev, mval, std_dev / mval)\n", 62 | " kernel_shap_nsamples.append(nsamples)\n", 63 | " kernel_shap_std.append(std_dev)\n", 64 | " kernel_shap_m.append(mval)\n", 65 | "\n", 66 | " nsamples = 1000 * M\n", 67 | " print(\"Sample Explainer\")\n", 68 | " print(nsamples)\n", 69 | " e = SamplingExplainer(f, X.mean(0).reshape(1,M))\n", 70 | " out = np.vstack([e.shap_values(X[:1,:], silent=True, nsamples=nsamples) for i in range(10)])\n", 71 | " std_dev = out.std(0)[:-1].mean()\n", 72 | " mval = np.abs(out.mean(0))[:-1].mean()\n", 73 | " print(std_dev, mval, std_dev / mval)\n", 74 | " ime_nsamples.append(nsamples)\n", 75 | " ime_std.append(std_dev)\n", 76 | " ime_m.append(mval)\n", 77 | "\n", 78 | " ime_std_lst.append(ime_std)\n", 79 | " ime_m_lst.append(ime_m)\n", 80 | " kernel_shap_std_lst.append(kernel_shap_std)\n", 81 | " kernel_shap_m_lst.append(kernel_shap_m)\n", 82 | "np.save(\"scratch/kernel_shap_std_lst\",np.array(kernel_shap_std_lst))\n", 83 | "np.save(\"scratch/ime_std_lst\",np.array(ime_std_lst))\n", 84 | "np.save(\"scratch/kernel_shap_m_lst\",np.array(kernel_shap_m_lst))\n", 85 | "np.save(\"scratch/ime_m_lst\",np.array(ime_m_lst))" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [] 94 | } 95 | ], 96 | "metadata": { 97 | "kernelspec": { 98 | "display_name": "Python 3", 99 | "language": "python", 100 | "name": "python3" 101 | }, 102 | "language_info": { 103 | "codemirror_mode": { 104 | "name": "ipython", 105 | "version": 3 106 | }, 107 | "file_extension": ".py", 108 | "mimetype": "text/x-python", 109 | "name": "python", 110 | "nbconvert_exporter": "python", 111 | "pygments_lexer": "ipython3", 112 | "version": "3.6.6" 113 | } 114 | }, 115 | "nbformat": 4, 116 | "nbformat_minor": 2 117 | } 118 | -------------------------------------------------------------------------------- /notebooks/simulation/Generate and Save Deviations.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Generate and Save Sampling(IME)/Kernel Explainer convergence details for a simulated data set" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import xgboost\n", 17 | "import numpy as np\n", 18 | "import shap\n", 19 | "import time\n", 20 | "from tqdm import tqdm\n", 21 | "import matplotlib.pylab as pl\n", 22 | "from shap import KernelExplainer\n", 23 | "from shap import SamplingExplainer\n", 24 | "N = 1000\n", 25 | "Ms = [20,30,40,50,60,70,80,90]\n", 26 | "\n", 27 | "np.random.seed(3011)\n", 28 | "ime_std_lst =[]\n", 29 | "ime_m_lst = []\n", 30 | "kernel_shap_std_lst =[]\n", 31 | "kernel_shap_m_lst = []\n", 32 | "for l in range(0,30):\n", 33 | " X_full = np.random.randn(N, 100)\n", 34 | " y = np.random.randn(N)\n", 35 | " kernel_shap_nsamples = []\n", 36 | " ime_nsamples = []\n", 37 | " tree_shap_std = []\n", 38 | " kernel_shap_std = []\n", 39 | " kernel_shap_m = []\n", 40 | " ime_std = []\n", 41 | " ime_m = []\n", 42 | " for M in Ms:\n", 43 | " print(\"\\nM\", M)\n", 44 | " X = X_full[:,:M]\n", 45 | " model = xgboost.train({\"silent\":1}, xgboost.DMatrix(X, y), 1000)\n", 46 | "\n", 47 | " def f(x):\n", 48 | " return model.predict(xgboost.DMatrix(x))\n", 49 | "\n", 50 | " e = shap.TreeExplainer(model)\n", 51 | " e.shap_values(X)\n", 52 | " tree_shap_std.append(0)\n", 53 | "\n", 54 | " print(\"Kernel Explainer\") \n", 55 | " nsamples = 1000 * M\n", 56 | " print(nsamples)\n", 57 | " e = KernelExplainer(f, X.mean(0).reshape(1,M))\n", 58 | " out = np.vstack([e.shap_values(X[:1,:], silent=True, nsamples=nsamples) for i in range(10)])\n", 59 | " std_dev = out.std(0)[:-1].mean()\n", 60 | " mval = np.abs(out.mean(0))[:-1].mean()\n", 61 | " print(std_dev, mval, std_dev / mval)\n", 62 | " kernel_shap_nsamples.append(nsamples)\n", 63 | " kernel_shap_std.append(std_dev)\n", 64 | " kernel_shap_m.append(mval)\n", 65 | "\n", 66 | " nsamples = 1000 * M\n", 67 | " print(\"Sample Explainer\")\n", 68 | " print(nsamples)\n", 69 | " e = SamplingExplainer(f, X.mean(0).reshape(1,M))\n", 70 | " out = np.vstack([e.shap_values(X[:1,:], silent=True, nsamples=nsamples) for i in range(10)])\n", 71 | " std_dev = out.std(0)[:-1].mean()\n", 72 | " mval = np.abs(out.mean(0))[:-1].mean()\n", 73 | " print(std_dev, mval, std_dev / mval)\n", 74 | " ime_nsamples.append(nsamples)\n", 75 | " ime_std.append(std_dev)\n", 76 | " ime_m.append(mval)\n", 77 | "\n", 78 | " ime_std_lst.append(ime_std)\n", 79 | " ime_m_lst.append(ime_m)\n", 80 | " kernel_shap_std_lst.append(kernel_shap_std)\n", 81 | " kernel_shap_m_lst.append(kernel_shap_m)\n", 82 | "np.save(\"scratch/kernel_shap_std_lst\",np.array(kernel_shap_std_lst))\n", 83 | "np.save(\"scratch/ime_std_lst\",np.array(ime_std_lst))\n", 84 | "np.save(\"scratch/kernel_shap_m_lst\",np.array(kernel_shap_m_lst))\n", 85 | "np.save(\"scratch/ime_m_lst\",np.array(ime_m_lst))" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [] 94 | } 95 | ], 96 | "metadata": { 97 | "kernelspec": { 98 | "display_name": "Python 3", 99 | "language": "python", 100 | "name": "python3" 101 | }, 102 | "language_info": { 103 | "codemirror_mode": { 104 | "name": "ipython", 105 | "version": 3 106 | }, 107 | "file_extension": ".py", 108 | "mimetype": "text/x-python", 109 | "name": "python", 110 | "nbconvert_exporter": "python", 111 | "pygments_lexer": "ipython3", 112 | "version": "3.6.6" 113 | } 114 | }, 115 | "nbformat": 4, 116 | "nbformat_minor": 2 117 | } 118 | -------------------------------------------------------------------------------- /notebooks/simulation/scratch/ime_m_lst.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/notebooks/simulation/scratch/ime_m_lst.npy -------------------------------------------------------------------------------- /notebooks/simulation/scratch/ime_std_lst.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/notebooks/simulation/scratch/ime_std_lst.npy -------------------------------------------------------------------------------- /notebooks/simulation/scratch/kernel_shap_m_lst.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/notebooks/simulation/scratch/kernel_shap_m_lst.npy -------------------------------------------------------------------------------- /notebooks/simulation/scratch/kernel_shap_std_lst.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/notebooks/simulation/scratch/kernel_shap_std_lst.npy -------------------------------------------------------------------------------- /notebooks/timeseries/Explain LSTM->XGB.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Explain LSTM->XGB on TBI signal data\n", 8 | "\n", 9 | "Explain XGBoost model trained on LSTM embedded data" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "import os\n", 19 | "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\"\n", 20 | "\n", 21 | "import numpy as np\n", 22 | "from tbi_downstream_prediction import split_data\n", 23 | "\n", 24 | "import keras\n", 25 | "from keras.utils import multi_gpu_model\n", 26 | "from keras.layers import Input, LSTM, Dense, Dropout\n", 27 | "from keras.models import Sequential, load_model, Model\n", 28 | "from matplotlib import cm, pyplot as plt\n", 29 | "from sklearn import metrics\n", 30 | "from os.path import expanduser as eu\n", 31 | "from os.path import isfile, join\n", 32 | "import numpy as np\n", 33 | "import random, time\n", 34 | "\n", 35 | "import tensorflow as tf\n", 36 | "from keras.backend.tensorflow_backend import set_session\n", 37 | "config = tf.ConfigProto(allow_soft_placement=True,gpu_options = tf.GPUOptions(allow_growth=True))\n", 38 | "set_session(tf.Session(config=config))\n", 39 | "\n", 40 | "PATH = \"/homes/gws/hughchen/phase/tbi_subset/\"\n", 41 | "DPATH = PATH+\"tbi/processed_data/hypoxemia/\"\n", 42 | "data_type = \"raw[top11]\"\n", 43 | "\n", 44 | "feat_lst = [\"ECGRATE\", \"ETCO2\", \"ETSEV\", \"ETSEVO\", \"FIO2\", \"NIBPD\", \"NIBPM\", \n", 45 | " \"NIBPS\",\"PEAK\", \"PEEP\", \"PIP\", \"RESPRATE\", \"SAO2\", \"TEMP1\", \"TV\"]\n", 46 | "\n", 47 | "# Exclude these features\n", 48 | "weird_feat_lst = [\"ETSEV\", \"PIP\", \"PEEP\", \"TV\"]\n", 49 | "feat_inds = np.array([feat_lst.index(feat) for feat in feat_lst if feat not in weird_feat_lst])\n", 50 | "feat_lst2 = [feat for feat in feat_lst if feat not in weird_feat_lst]\n", 51 | "\n", 52 | "y_tbi = np.load(DPATH+\"tbiy.npy\")\n", 53 | "X_tbi = np.load(DPATH+\"X_tbi_imp_standard.npy\")\n", 54 | "\n", 55 | "X_tbi2 = X_tbi[:,feat_inds,:]\n", 56 | "(X_test, y_test, X_valid, y_valid, X_train, y_train) = split_data(DPATH,X_tbi2,y_tbi,flatten=False)\n", 57 | "\n", 58 | "PATH = \"/homes/gws/hughchen/phase/tbi_subset/\"\n", 59 | "RESULTPATH = PATH+\"results/\"\n", 60 | "label_type = \"desat_bool92_5_nodesat\"\n", 61 | "lstm_type = \"biglstmdropoutv3_{}\".format(label_type)\n", 62 | "RESDIR = '{}{}/'.format(RESULTPATH, lstm_type)\n", 63 | "if not os.path.exists(RESDIR): os.makedirs(RESDIR)\n", 64 | "GPUNUM = len(os.environ[\"CUDA_VISIBLE_DEVICES\"].split(\",\"))" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "### Load model and generate independent explanations" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "import shap\n", 81 | "import xgboost as xgb\n", 82 | "\n", 83 | "from tbi_downstream_prediction import *\n", 84 | "PATH = \"/homes/gws/hughchen/phase/tbi_subset/\"\n", 85 | "DPATH = PATH+\"tbi/processed_data/hypoxemia/\"\n", 86 | "RESULTPATH = PATH+\"results/\"\n", 87 | "\n", 88 | "# Load data\n", 89 | "X_train = np.load(DPATH+\"X_train_embed_lstmbigdropoutv3_50n_arr.npy\",mmap_mode=\"r\")\n", 90 | "X_valid = np.load(DPATH+\"X_valid_embed_lstmbigdropoutv3_50n_arr.npy\",mmap_mode=\"r\")\n", 91 | "X_test = np.load(DPATH+\"X_test_embed_lstmbigdropoutv3_50n_arr.npy\",mmap_mode=\"r\")\n", 92 | "\n", 93 | "# Set important variables\n", 94 | "label_type = \"desat_bool92_5_nodesat\"; eta = 0.02\n", 95 | "hosp_data = \"tbi\"; data_type = \"lstm_big_50n[top11]\"\n", 96 | "mod_type = \"xgb_{}_eta{}\".format(label_type,eta)\n", 97 | "\n", 98 | "# Set up result directory\n", 99 | "RESDIR = '{}results/{}/'.format(PATH, mod_type)\n", 100 | "if not os.path.exists(RESDIR): os.makedirs(RESDIR)\n", 101 | "\n", 102 | "# Set parameters to train model\n", 103 | "param = {'max_depth':6, 'eta':eta, 'subsample':0.5, 'gamma':1.0, \n", 104 | " 'min_child_weight':10, 'base_score':y_train.mean(), \n", 105 | " 'objective':'binary:logistic', 'eval_metric':[\"logloss\"]}\n", 106 | "\n", 107 | "save_path = RESDIR+\"hosp{}_data/{}/\".format(hosp_data,data_type)\n", 108 | "bst = xgb.Booster()\n", 109 | "bst.load_model(save_path+'mod_eta{}.model'.format(param['eta']))\n", 110 | "X_tbi2 = X_tbi[:,feat_inds,:]\n", 111 | "(_, _, _, _, X_train_raw, _) = split_data(DPATH,X_tbi2,y_tbi,flatten=False)\n", 112 | "\n", 113 | "# Generate explanations\n", 114 | "import numpy as np\n", 115 | "np.random.seed(10)\n", 116 | "background_inds = np.random.choice(np.arange(0,X_train.shape[0]),1000)\n", 117 | "X_train = np.load(DPATH+\"X_train_embed_lstmbigdropoutv2_arr.npy\")\n", 118 | "X_train_background = X_train[background_inds]\n", 119 | "ind_explainer = shap.TreeExplainer(bst,data=X_train_background,feature_dependence=\"independent\")\n", 120 | "X_train_ind_explanations = ind_explainer.shap_values(X_train)\n", 121 | "np.save(save_path+\"X_train_explanations_ind\",X_train_ind_explanations)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "metadata": {}, 127 | "source": [ 128 | "#### Plot the aggregated explanations" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": null, 134 | "metadata": {}, 135 | "outputs": [], 136 | "source": [ 137 | "X_train_ind_explanations_agg = np.transpose(np.vstack([X_train_ind_explanations[:,i*50:(i+1)*50].sum(1) for i in range(len(feat_lst2))]))\n", 138 | "shap.summary_plot(X_train_ind_explanations_agg,X_train_raw.sum(1),feature_names=feat_lst2)" 139 | ] 140 | } 141 | ], 142 | "metadata": { 143 | "kernelspec": { 144 | "display_name": "Python 3", 145 | "language": "python", 146 | "name": "python3" 147 | }, 148 | "language_info": { 149 | "codemirror_mode": { 150 | "name": "ipython", 151 | "version": 3 152 | }, 153 | "file_extension": ".py", 154 | "mimetype": "text/x-python", 155 | "name": "python", 156 | "nbconvert_exporter": "python", 157 | "pygments_lexer": "ipython3", 158 | "version": "3.6.6" 159 | } 160 | }, 161 | "nbformat": 4, 162 | "nbformat_minor": 2 163 | } 164 | -------------------------------------------------------------------------------- /notebooks/timeseries/Explain XGB.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Explain XGB on TBI signal data\n", 8 | "\n", 9 | "Explain XGBoost model trained on raw data" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "from tbi_downstream_prediction import *\n", 19 | "PATH = \"/homes/gws/hughchen/phase/tbi_subset/\"\n", 20 | "DPATH = PATH+\"tbi/processed_data/hypoxemia/\"\n", 21 | "RESULTPATH = PATH+\"results/\"\n", 22 | "\n", 23 | "# Set important variables\n", 24 | "label_type = \"desat_bool92_5_nodesat\"; eta = 0.02\n", 25 | "hosp_data = \"tbi\"; data_type = \"raw[top11]\"\n", 26 | "mod_type = \"xgb_{}_eta{}\".format(label_type,eta)\n", 27 | "\n", 28 | "# Set up result directory\n", 29 | "RESDIR = '{}results/{}/'.format(PATH, mod_type)\n", 30 | "if not os.path.exists(RESDIR): os.makedirs(RESDIR)\n", 31 | "\n", 32 | "# Load tbi data\n", 33 | "y_tbi = np.load(DPATH+\"tbiy.npy\",mmap_mode=\"r\")\n", 34 | "X_tbi = np.load(DPATH+\"X_tbi_imp_standard.npy\",mmap_mode=\"r\")\n", 35 | "\n", 36 | "feats = [\"ECGRATE\", \"ETCO2\", \"ETSEV\", \"ETSEVO\", \"FIO2\", \"NIBPD\", \"NIBPM\", \n", 37 | " \"NIBPS\",\"PEAK\", \"PEEP\", \"PIP\", \"RESPRATE\", \"SAO2\", \"TEMP1\", \"TV\"]\n", 38 | "weird_feats = [\"ETSEV\", \"PIP\", \"PEEP\", \"TV\"]\n", 39 | "feat_inds = np.array([feats.index(feat) for feat in feats if feat not in weird_feats]) \n", 40 | "X_tbi2 = X_tbi[:,feat_inds,:]\n", 41 | "(X_test, y_test, X_valid, y_valid, X_train, y_train) = split_data(DPATH,X_tbi2,y_tbi)\n", 42 | "feat_lst2 = [feat for feat in feats if feat not in weird_feats]\n", 43 | "\n", 44 | "# Set parameters to train model\n", 45 | "param = {'max_depth':6, 'eta':eta, 'subsample':0.5, 'gamma':1.0, \n", 46 | " 'min_child_weight':10, 'base_score':y_train.mean(), \n", 47 | " 'objective':'binary:logistic', 'eval_metric':[\"logloss\"]}\n", 48 | "\n", 49 | "# Generate attributions\n", 50 | "import shap\n", 51 | "save_path = RESDIR+\"hosp{}_data/{}/\".format(hosp_data,data_type)\n", 52 | "if DEBUG: print(\"[DEBUG] Loading model from {}\".format(save_path))\n", 53 | "bst = xgb.Booster()\n", 54 | "bst.load_model(save_path+'mod_eta{}.model'.format(param['eta']))\n", 55 | "\n", 56 | "np.random.seed(10)\n", 57 | "background_inds = np.random.choice(np.arange(0,X_train.shape[0]),1000)\n", 58 | "X_train_background = X_train[background_inds]\n", 59 | "ind_explainer = shap.TreeExplainer(bst,data=X_train_background,feature_dependence=\"independent\")\n", 60 | "X_train_ind_explanations = ind_explainer.shap_values(X_train)\n", 61 | "np.save(save_path+\"X_train_explanations_ind\",X_train_ind_explanations)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "#### Plot the attributions for each time point" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "save_path = RESDIR+\"hosp{}_data/{}/\".format(hosp_data,data_type)\n", 78 | "import itertools\n", 79 | "feat_lst2_expanded = [[f+\"_\"+str(i) for i in range(0,60)] for f in feat_lst2]\n", 80 | "feat_lst2_expanded = list(itertools.chain.from_iterable(feat_lst2_expanded))\n", 81 | "shap.summary_plot(X_train_ind_explanations,features=X_train,feature_names=feat_lst2_expanded,max_display=10)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "#### Plot the aggregated attributions" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "X_train_explanations_agg = np.transpose(np.vstack([X_train_ind_explanations[:,i*60:(i+1)*60].sum(1) for i in range(len(feat_lst2))]))\n", 98 | "X_train_agg = np.transpose(np.vstack([X_train[:,i*60:(i+1)*60].sum(1) for i in range(len(feat_lst2))]))\n", 99 | "shap.summary_plot(X_train_explanations_agg,features=X_train_agg,feature_names=feat_lst2)" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "### Plot the time series signal and attribution for a single sample" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "import os\n", 116 | "import numpy as np\n", 117 | "DPATH = \"/homes/gws/hughchen/phase/tbi_subset/tbi/processed_data/hypoxemia/\"\n", 118 | "\n", 119 | "feat_lst = [\"ECGRATE\", \"ETCO2\", \"ETSEV\", \"ETSEVO\", \"FIO2\", \"NIBPD\", \"NIBPM\", \n", 120 | " \"NIBPS\",\"PEAK\", \"PEEP\", \"PIP\", \"RESPRATE\", \"SAO2\", \"TEMP1\", \"TV\"]\n", 121 | "\n", 122 | "# The data used for training is normalized. Here we are getting the unnormalized\n", 123 | "# but imputed data for the sake of plotting.\n", 124 | "X_tbi = np.load(DPATH+\"tbiX.npy\",mmap_mode=\"r\")\n", 125 | "\n", 126 | "# Impute missing data\n", 127 | "def impute_data(dataX):\n", 128 | " imp_val = np.median(dataX[dataX != 0])\n", 129 | " if np.isnan(imp_val): imp_val = 0\n", 130 | " dataX[dataX == 0] = imp_val\n", 131 | " return(dataX)\n", 132 | "\n", 133 | "X_tbi_imp = []\n", 134 | "for feat in feat_lst:\n", 135 | " fpath = \"/homes/gws/hughchen/RNN/LSTM_Feature/code/\"\n", 136 | " fpath += \"min5_data/{}minimum5/hospital_0/raw/\".format(feat)\n", 137 | " fname = [f for f in os.listdir(fpath) if feat+\".npy\" in f and \"X_train_val\" in f][0]\n", 138 | " X_trval = np.load(fpath+fname)\n", 139 | " X_tbi_curr = np.copy(X_tbi[:,feat_lst.index(feat),:])\n", 140 | " if feat == \"TV\": X_tbi_curr = X_tbi_curr/1000.0\n", 141 | "\n", 142 | " X_trval_imp = impute_data(X_trval)\n", 143 | " X_tbi_curr[np.isnan(X_tbi_curr)] = 0\n", 144 | " X_tbi_curr_imp = impute_data(X_tbi_curr)\n", 145 | " X_tbi_imp.append(X_tbi_curr_imp)\n", 146 | "\n", 147 | "X_tbi_imp = np.stack(X_tbi_imp,axis=1)\n", 148 | "\n", 149 | "(_, _, _, _, X_train_raw_imp, _) = split_data(DPATH,X_tbi_imp,y_tbi,flatten=False)\n", 150 | "\n", 151 | "# FIO2_59_ind = ((60*(feat_lst2.index(\"FIO2\")+1))-1)\n", 152 | "# FIO2_59_exp = X_train_ind_explanations[:,FIO2_59_ind]\n", 153 | "# best_samp_ind = np.where(FIO2_59_exp==FIO2_59_exp.max())[0][0]\n", 154 | "import matplotlib.pyplot as plt\n", 155 | "plt.rcParams[\"figure.figsize\"] = (5,2)\n", 156 | "\n", 157 | "samp_ind = np.random.choice(X_train.shape[0])\n", 158 | "# samp_ind = 189432\n", 159 | "samp_ind = 14898\n", 160 | "\n", 161 | "signame = \"TEMP1\"\n", 162 | "ind = feat_lst2.index(signame)\n", 163 | "# plt.plot(X_train_raw_imp[feats.index(signame)][samp_ind])\n", 164 | "plt.plot(X_train_raw_imp[samp_ind,:,feats.index(signame)])\n", 165 | "# plt.plot(X_train[samp_ind,(60*ind):(60*(ind+1))])\n", 166 | "plt.ylabel(signame)\n", 167 | "plt.savefig(\"fig/tempvalu{}.pdf\".format(samp_ind))\n", 168 | "plt.show()\n", 169 | "plt.plot(X_train_ind_explanations[samp_ind,(60*ind):(60*(ind+1))])\n", 170 | "plt.savefig(\"fig/tempattr{}.pdf\".format(samp_ind))\n", 171 | "plt.show()\n", 172 | "\n", 173 | "signame = \"SAO2\"\n", 174 | "ind = feat_lst2.index(signame)\n", 175 | "plt.plot(X_train_raw_imp[samp_ind,:,feats.index(signame)])\n", 176 | "plt.ylabel(signame)\n", 177 | "plt.savefig(\"fig/sao2valu{}.pdf\".format(samp_ind))\n", 178 | "plt.show()\n", 179 | "plt.plot(X_train_ind_explanations[samp_ind,(60*ind):(60*(ind+1))])\n", 180 | "plt.savefig(\"fig/sao2attr{}.pdf\".format(samp_ind))\n", 181 | "plt.show()\n", 182 | "\n", 183 | "signame = \"FIO2\"\n", 184 | "ind = feat_lst2.index(signame)\n", 185 | "# plt.plot(X_train_raw_imp[feats.index(signame)][samp_ind])\n", 186 | "plt.plot(X_train_raw_imp[samp_ind,:,feats.index(signame)])\n", 187 | "# plt.plot(X_train[samp_ind,(60*ind):(60*(ind+1))])\n", 188 | "plt.ylabel(signame)\n", 189 | "plt.savefig(\"fig/fio2valu{}.pdf\".format(samp_ind))\n", 190 | "plt.show()\n", 191 | "plt.plot(X_train_ind_explanations[samp_ind,(60*ind):(60*(ind+1))])\n", 192 | "plt.savefig(\"fig/fio2attr{}.pdf\".format(samp_ind))\n", 193 | "plt.show()" 194 | ] 195 | } 196 | ], 197 | "metadata": { 198 | "kernelspec": { 199 | "display_name": "Python 3", 200 | "language": "python", 201 | "name": "python3" 202 | }, 203 | "language_info": { 204 | "codemirror_mode": { 205 | "name": "ipython", 206 | "version": 3 207 | }, 208 | "file_extension": ".py", 209 | "mimetype": "text/x-python", 210 | "name": "python", 211 | "nbconvert_exporter": "python", 212 | "pygments_lexer": "ipython3", 213 | "version": "3.6.6" 214 | } 215 | }, 216 | "nbformat": 4, 217 | "nbformat_minor": 2 218 | } 219 | -------------------------------------------------------------------------------- /notebooks/timeseries/Run LSTM->XGB.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Train LSTM->XGB on TBI signal data\n", 8 | "\n", 9 | "Code to train an XGBoost model on the TBI data (embedded by the LSTM) to predict hypoxemia in the future (low SAO2).\n", 10 | "\n", 11 | "Note that the data is private and we are unable to make it publicly available in this repo." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "### Load data and setup paths" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "import os\n", 28 | "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\"\n", 29 | "\n", 30 | "import numpy as np\n", 31 | "from tbi_downstream_prediction import split_data\n", 32 | "\n", 33 | "import keras\n", 34 | "from keras.utils import multi_gpu_model\n", 35 | "from keras.layers import Input, LSTM, Dense, Dropout\n", 36 | "from keras.models import Sequential, load_model, Model\n", 37 | "from matplotlib import cm, pyplot as plt\n", 38 | "from sklearn import metrics\n", 39 | "from os.path import expanduser as eu\n", 40 | "from os.path import isfile, join\n", 41 | "import numpy as np\n", 42 | "import random, time\n", 43 | "\n", 44 | "import tensorflow as tf\n", 45 | "from keras.backend.tensorflow_backend import set_session\n", 46 | "config = tf.ConfigProto(allow_soft_placement=True,gpu_options = tf.GPUOptions(allow_growth=True))\n", 47 | "set_session(tf.Session(config=config))\n", 48 | "\n", 49 | "PATH = \"/homes/gws/hughchen/phase/tbi_subset/\"\n", 50 | "DPATH = PATH+\"tbi/processed_data/hypoxemia/\"\n", 51 | "data_type = \"raw[top11]\"\n", 52 | "\n", 53 | "feat_lst = [\"ECGRATE\", \"ETCO2\", \"ETSEV\", \"ETSEVO\", \"FIO2\", \"NIBPD\", \"NIBPM\", \n", 54 | " \"NIBPS\",\"PEAK\", \"PEEP\", \"PIP\", \"RESPRATE\", \"SAO2\", \"TEMP1\", \"TV\"]\n", 55 | "\n", 56 | "# Exclude these features\n", 57 | "weird_feat_lst = [\"ETSEV\", \"PIP\", \"PEEP\", \"TV\"]\n", 58 | "feat_inds = np.array([feat_lst.index(feat) for feat in feat_lst if feat not in weird_feat_lst])\n", 59 | "feat_lst2 = [feat for feat in feat_lst if feat not in weird_feat_lst]\n", 60 | "\n", 61 | "y_tbi = np.load(DPATH+\"tbiy.npy\")\n", 62 | "X_tbi = np.load(DPATH+\"X_tbi_imp_standard.npy\")\n", 63 | "\n", 64 | "X_tbi2 = X_tbi[:,feat_inds,:]\n", 65 | "(X_test, y_test, X_valid, y_valid, X_train, y_train) = split_data(DPATH,X_tbi2,y_tbi,flatten=False)\n", 66 | "\n", 67 | "PATH = \"/homes/gws/hughchen/phase/tbi_subset/\"\n", 68 | "RESULTPATH = PATH+\"results/\"\n", 69 | "label_type = \"desat_bool92_5_nodesat\"\n", 70 | "lstm_type = \"biglstmdropoutv3_{}\".format(label_type)\n", 71 | "RESDIR = '{}{}/'.format(RESULTPATH, lstm_type)\n", 72 | "if not os.path.exists(RESDIR): os.makedirs(RESDIR)\n", 73 | "GPUNUM = len(os.environ[\"CUDA_VISIBLE_DEVICES\"].split(\",\"))" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "### Load best LSTM model from \"Run LSTM\" and create an embedding model" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "opt_name=\"rmsprop\";lr=0.001;drop=0.5;b_size=1000;epoch_num=200\n", 90 | "\n", 91 | "# Fixed hyperpara\n", 92 | "print(\"[PROGRESS] Starting create_train_model()\")\n", 93 | "lookback = 60; h1 = 50; h2 = 50\n", 94 | "loss_func = \"binary_crossentropy\"\n", 95 | "\n", 96 | "# Form the model name (for saving the model)\n", 97 | "mod_name = \"multivariate_biglstmdropoutv3_{}n_{}n_{}ep\".format(h1,h2,epoch_num)\n", 98 | "mod_name += \"_{}opt_{}lr\".format(opt_name,lr)\n", 99 | "mod_name += \"_{}drop_{}bs\".format(drop,b_size)\n", 100 | "MODDIR = PATH+\"models/tune_biglstm/\"+mod_name+\"/\"\n", 101 | "\n", 102 | "# Load the best model (in terms of validation performance)\n", 103 | "min_mod = load_min_model_helper(MODDIR)\n", 104 | "\n", 105 | "########## Form Model/Data #########\n", 106 | "min_mod_weights = min_mod.get_weights()\n", 107 | "\n", 108 | "X_train_lst = []; X_valid_lst = []; X_test_lst = []\n", 109 | "sig_lst = []; encoded_lst = []\n", 110 | "\n", 111 | "for i in range(0,len(feat_lst2)):\n", 112 | " X_train_lst.append(X_train[:,:,i:(i+1)])\n", 113 | " X_valid_lst.append(X_valid[:,:,i:(i+1)])\n", 114 | " X_test_lst.append(X_test[:,:,i:(i+1)])\n", 115 | "\n", 116 | " sig = Input(shape=(lookback,1))\n", 117 | " lstm1 = LSTM(h1, recurrent_dropout=drop, return_sequences=True, \n", 118 | " weights=min_mod_weights[(i*3):((i+1)*3)])\n", 119 | " lstm2 = LSTM(h2, recurrent_dropout=drop, dropout=drop,\n", 120 | " weights=min_mod_weights[(33+(i*3)):(33+((i+1)*3))])\n", 121 | "\n", 122 | " encoded = lstm2(lstm1(sig))\n", 123 | " sig_lst.append(sig); encoded_lst.append(encoded)\n", 124 | "\n", 125 | "model = Model(inputs=sig_lst, outputs=encoded_lst)\n", 126 | "opt = \"rmsprop\"; loss=\"binary_crossentropy\"\n", 127 | "if GPUNUM > 1: model = multi_gpu_model(model,gpus=GPUNUM)\n", 128 | "model.compile(optimizer=opt, loss=loss_func)\n", 129 | "\n", 130 | "# Create embeddings\n", 131 | "X_train_embed_lst = model.predict(X_train_lst)\n", 132 | "X_valid_embed_lst = model.predict(X_valid_lst)\n", 133 | "X_test_embed_lst = model.predict(X_test_lst)\n", 134 | "\n", 135 | "np.save(DPATH+\"X_train_embed_lstmbigdropoutv3_50n_arr\",np.concatenate(X_train_embed_lst,1))\n", 136 | "np.save(DPATH+\"X_valid_embed_lstmbigdropoutv3_50n_arr\",np.concatenate(X_valid_embed_lst,1))\n", 137 | "np.save(DPATH+\"X_test_embed_lstmbigdropoutv3_50n_arr\", np.concatenate(X_test_embed_lst,1))" 138 | ] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": {}, 143 | "source": [ 144 | "### Load embedded data and train XGB model" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": null, 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "from tbi_downstream_prediction import *\n", 154 | "PATH = \"/homes/gws/hughchen/phase/tbi_subset/\"\n", 155 | "DPATH = PATH+\"tbi/processed_data/hypoxemia/\"\n", 156 | "RESULTPATH = PATH+\"results/\"\n", 157 | "\n", 158 | "# Load data\n", 159 | "X_train = np.load(DPATH+\"X_train_embed_lstmbigdropoutv3_50n_arr.npy\",mmap_mode=\"r\")\n", 160 | "X_valid = np.load(DPATH+\"X_valid_embed_lstmbigdropoutv3_50n_arr.npy\",mmap_mode=\"r\")\n", 161 | "X_test = np.load(DPATH+\"X_test_embed_lstmbigdropoutv3_50n_arr.npy\",mmap_mode=\"r\")\n", 162 | "\n", 163 | "# Set important variables\n", 164 | "label_type = \"desat_bool92_5_nodesat\"; eta = 0.02\n", 165 | "hosp_data = \"tbi\"; data_type = \"lstm_big_50n[top11]\"\n", 166 | "mod_type = \"xgb_{}_eta{}\".format(label_type,eta)\n", 167 | "\n", 168 | "# Set up result directory\n", 169 | "RESDIR = '{}results/{}/'.format(PATH, mod_type)\n", 170 | "if not os.path.exists(RESDIR): os.makedirs(RESDIR)\n", 171 | "\n", 172 | "# Set parameters to train model\n", 173 | "param = {'max_depth':6, 'eta':eta, 'subsample':0.5, 'gamma':1.0, \n", 174 | " 'min_child_weight':10, 'base_score':y_train.mean(), \n", 175 | " 'objective':'binary:logistic', 'eval_metric':[\"logloss\"]}\n", 176 | "\n", 177 | "# Train and save xgb model\n", 178 | "train_save_xgb_model(X_train,y_train,X_valid,y_valid,RESDIR,\n", 179 | " param,hosp_data,data_type)\n", 180 | "\n", 181 | "# Test xgb model\n", 182 | "test_xgb_model(X_test,y_test,RESDIR,param,hosp_data,data_type)" 183 | ] 184 | } 185 | ], 186 | "metadata": { 187 | "kernelspec": { 188 | "display_name": "Python 3", 189 | "language": "python", 190 | "name": "python3" 191 | }, 192 | "language_info": { 193 | "codemirror_mode": { 194 | "name": "ipython", 195 | "version": 3 196 | }, 197 | "file_extension": ".py", 198 | "mimetype": "text/x-python", 199 | "name": "python", 200 | "nbconvert_exporter": "python", 201 | "pygments_lexer": "ipython3", 202 | "version": "3.6.6" 203 | } 204 | }, 205 | "nbformat": 4, 206 | "nbformat_minor": 2 207 | } 208 | -------------------------------------------------------------------------------- /notebooks/timeseries/Run LSTM.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Train LSTM on TBI signal data\n", 8 | "\n", 9 | "Code to train an LSTM on the TBI data to predict hypoxemia in the future (low SAO2).\n", 10 | "\n", 11 | "Note that the data is private and we are unable to make it publicly available in this repo." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import os\n", 21 | "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1,4\"\n", 22 | "\n", 23 | "import numpy as np\n", 24 | "from tbi_downstream_prediction import split_data\n", 25 | "\n", 26 | "import keras\n", 27 | "from keras.utils import multi_gpu_model\n", 28 | "from keras.layers import Input, LSTM, Dense, Dropout\n", 29 | "from keras.models import Sequential, load_model, Model\n", 30 | "from matplotlib import cm, pyplot as plt\n", 31 | "from sklearn import metrics\n", 32 | "from os.path import expanduser as eu\n", 33 | "from os.path import isfile, join\n", 34 | "import numpy as np\n", 35 | "import random, time\n", 36 | "\n", 37 | "import tensorflow as tf\n", 38 | "from keras.backend.tensorflow_backend import set_session\n", 39 | "config = tf.ConfigProto(allow_soft_placement=True,gpu_options = tf.GPUOptions(allow_growth=True))\n", 40 | "set_session(tf.Session(config=config))\n", 41 | "GPUNUM = len(os.environ[\"CUDA_VISIBLE_DEVICES\"].split(\",\"))\n", 42 | "\n", 43 | "PATH = \"/homes/gws/hughchen/phase/tbi_subset/\"\n", 44 | "DPATH = PATH+\"tbi/processed_data/hypoxemia/\"\n", 45 | "data_type = \"raw[top11]\"\n", 46 | "\n", 47 | "feat_lst = [\"ECGRATE\", \"ETCO2\", \"ETSEV\", \"ETSEVO\", \"FIO2\", \"NIBPD\", \"NIBPM\", \n", 48 | " \"NIBPS\",\"PEAK\", \"PEEP\", \"PIP\", \"RESPRATE\", \"SAO2\", \"TEMP1\", \"TV\"]\n", 49 | "\n", 50 | "# Exclude these features\n", 51 | "exclude_feat_lst = [\"ETSEV\", \"PIP\", \"PEEP\", \"TV\"]\n", 52 | "feat_inds = np.array([feat_lst.index(feat) for feat in feat_lst if feat not in exclude_feat_lst])\n", 53 | "feat_lst2 = [feat for feat in feat_lst if feat not in exclude_feat_lst]\n", 54 | "\n", 55 | "y_tbi = np.load(DPATH+\"tbiy.npy\")\n", 56 | "X_tbi = np.load(DPATH+\"X_tbi_imp_standard.npy\")\n", 57 | "\n", 58 | "X_tbi2 = X_tbi[:,feat_inds,:]\n", 59 | "(X_test, y_test, X_valid, y_valid, X_train, y_train) = split_data(DPATH,X_tbi2,y_tbi,flatten=False)\n", 60 | "\n", 61 | "PATH = \"/homes/gws/hughchen/phase/tbi_subset/\"\n", 62 | "RESULTPATH = PATH+\"results/\"\n", 63 | "label_type = \"desat_bool92_5_nodesat\"\n", 64 | "lstm_type = \"biglstmdropoutv3_{}\".format(label_type)\n", 65 | "RESDIR = '{}{}/'.format(RESULTPATH, lstm_type)\n", 66 | "if not os.path.exists(RESDIR): os.makedirs(RESDIR)" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "### Hyperparameter tuning" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "metadata": {}, 80 | "outputs": [], 81 | "source": [ 82 | "# Set of hyperparameters to tune over\n", 83 | "node_lst = [50, 100, 200]\n", 84 | "opt_lst = [\"rmsprop\", \"adam\", \"sgd\"]\n", 85 | "lr_lst = [0.01, 0.001, 0.0001]\n", 86 | "drop_lst = [0.3, 0.5, 0.7]\n", 87 | "ISTUNE = True\n", 88 | "\n", 89 | "# Greedily find best opt\n", 90 | "min_loss_lst = [\n", 91 | " create_train_model(opt_name=\"rmsprop\",lr=0.001,drop=0.5,\n", 92 | " b_size=1000,nodesize=para)\n", 93 | " for para in node_lst\n", 94 | "]\n", 95 | "best_nodesize = node_lst[min_loss_lst.index(min(min_loss_lst))]\n", 96 | "print(\"best best_nodesize: {}\".format(best_nodesize))\n", 97 | "\n", 98 | "# Greedily find best opt\n", 99 | "min_loss_lst = [\n", 100 | " create_train_model(opt_name=para,lr=0.001,drop=0.5,\n", 101 | " b_size=1000,nodesize=best_nodesize)\n", 102 | " for para in opt_lst\n", 103 | "]\n", 104 | "best_optname = opt_lst[min_loss_lst.index(min(min_loss_lst))]\n", 105 | "print(\"best optname: {}\".format(best_optname))\n", 106 | "\n", 107 | "# Greedily find best lr\n", 108 | "min_loss_lst = [\n", 109 | " create_train_model(opt_name=best_optname,lr=para,drop=0.5,\n", 110 | " b_size=1000,nodesize=best_nodesize)\n", 111 | " for para in lr_lst\n", 112 | "]\n", 113 | "best_lr = lr_lst[min_loss_lst.index(min(min_loss_lst))]\n", 114 | "print(\"best lr: {}\".format(best_lr))\n", 115 | "\n", 116 | "# Greedily find best dropout\n", 117 | "min_loss_lst = [\n", 118 | " create_train_model(opt_name=best_optname,lr=best_lr,drop=para,\n", 119 | " b_size=1000,nodesize=best_nodesize)\n", 120 | " for para in drop_lst\n", 121 | "]\n", 122 | "best_drop = drop_lst[min_loss_lst.index(min(min_loss_lst))]\n", 123 | "print(\"best drop: {}\".format(best_drop))" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": {}, 129 | "source": [ 130 | "### Train the model with the best hyperparameters" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "create_train_model(opt_name=best_optname,lr=best_lr,drop=best_drop,\n", 140 | " b_size=1000,nodesize=best_nodesize,epoch_num=200,\n", 141 | " is_tune=False)\n", 142 | "eval_test_model(opt_name=best_optname,lr=best_lr,drop=best_drop,\n", 143 | " b_size=1000,nodesize=best_nodesize,epoch_num=200)" 144 | ] 145 | } 146 | ], 147 | "metadata": { 148 | "kernelspec": { 149 | "display_name": "Python 3", 150 | "language": "python", 151 | "name": "python3" 152 | }, 153 | "language_info": { 154 | "codemirror_mode": { 155 | "name": "ipython", 156 | "version": 3 157 | }, 158 | "file_extension": ".py", 159 | "mimetype": "text/x-python", 160 | "name": "python", 161 | "nbconvert_exporter": "python", 162 | "pygments_lexer": "ipython3", 163 | "version": "3.6.6" 164 | } 165 | }, 166 | "nbformat": 4, 167 | "nbformat_minor": 2 168 | } 169 | -------------------------------------------------------------------------------- /notebooks/timeseries/Run XGB.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Train XGB on TBI signal data\n", 8 | "\n", 9 | "Code to train an XGBoost model on the TBI data to predict hypoxemia in the future (low SAO2).\n", 10 | "\n", 11 | "Note that the data is private and we are unable to make it publicly available in this repo." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "from tbi_downstream_prediction import *\n", 21 | "PATH = \"/homes/gws/hughchen/phase/tbi_subset/\"\n", 22 | "DPATH = PATH+\"tbi/processed_data/hypoxemia/\"\n", 23 | "RESULTPATH = PATH+\"results/\"\n", 24 | "\n", 25 | "# Set important variables\n", 26 | "label_type = \"desat_bool92_5_nodesat\"; eta = 0.02\n", 27 | "hosp_data = \"tbi\"; data_type = \"raw[top11]\"\n", 28 | "mod_type = \"xgb_{}_eta{}\".format(label_type,eta)\n", 29 | "\n", 30 | "# Set up result directory\n", 31 | "RESDIR = '{}results/{}/'.format(PATH, mod_type)\n", 32 | "if not os.path.exists(RESDIR): os.makedirs(RESDIR)\n", 33 | "\n", 34 | "# Load tbi data\n", 35 | "y_tbi = np.load(DPATH+\"tbiy.npy\",mmap_mode=\"r\")\n", 36 | "X_tbi = np.load(DPATH+\"X_tbi_imp_standard.npy\",mmap_mode=\"r\")\n", 37 | "\n", 38 | "feats = [\"ECGRATE\", \"ETCO2\", \"ETSEV\", \"ETSEVO\", \"FIO2\", \"NIBPD\", \"NIBPM\", \n", 39 | " \"NIBPS\",\"PEAK\", \"PEEP\", \"PIP\", \"RESPRATE\", \"SAO2\", \"TEMP1\", \"TV\"]\n", 40 | "weird_feats = [\"ETSEV\", \"PIP\", \"PEEP\", \"TV\"]\n", 41 | "feat_inds = np.array([feats.index(feat) for feat in feats if feat not in weird_feats]) \n", 42 | "X_tbi2 = X_tbi[:,feat_inds,:]\n", 43 | "(X_test, y_test, X_valid, y_valid, X_train, y_train) = split_data(DPATH,X_tbi2,y_tbi)\n", 44 | "feat_lst2 = [feat for feat in feats if feat not in weird_feats]\n", 45 | "\n", 46 | "# Set parameters to train model\n", 47 | "param = {'max_depth':6, 'eta':eta, 'subsample':0.5, 'gamma':1.0, \n", 48 | " 'min_child_weight':10, 'base_score':y_train.mean(), \n", 49 | " 'objective':'binary:logistic', 'eval_metric':[\"logloss\"]}" 50 | ] 51 | } 52 | ], 53 | "metadata": { 54 | "kernelspec": { 55 | "display_name": "Python 3", 56 | "language": "python", 57 | "name": "python3" 58 | }, 59 | "language_info": { 60 | "codemirror_mode": { 61 | "name": "ipython", 62 | "version": 3 63 | }, 64 | "file_extension": ".py", 65 | "mimetype": "text/x-python", 66 | "name": "python", 67 | "nbconvert_exporter": "python", 68 | "pygments_lexer": "ipython3", 69 | "version": "3.6.6" 70 | } 71 | }, 72 | "nbformat": 4, 73 | "nbformat_minor": 2 74 | } 75 | -------------------------------------------------------------------------------- /notebooks/timeseries/tbi_downstream_prediction.py: -------------------------------------------------------------------------------- 1 | import sklearn.metrics as metrics 2 | import xgboost as xgb 3 | import numpy as np 4 | import sys, os 5 | 6 | DEBUG = False 7 | TBI_FEATS = ["ECGRATE", "ETCO2", "ETSEV", "ETSEVO", "FIO2", "NIBPD", "NIBPM", 8 | "NIBPS","PEAK", "PEEP", "PIP", "RESPRATE", "SAO2", "TEMP1", "TV"] 9 | 10 | def split_data(DPATH,X_tbi,y_tbi,IDPATH=None,flatten=True): 11 | if IDPATH is None: IDPATH = DPATH 12 | 13 | sample_tbi_ids = np.load(DPATH+"tbiid.npy",mmap_mode="r") 14 | unique_tbi_ids = np.unique(sample_tbi_ids) 15 | 16 | # Test and trval splits 17 | test_tbi_ids = np.load(IDPATH+"test_tbi_ids.npy") 18 | trval_tbi_ids = np.load(IDPATH+"trval_tbi_ids.npy") 19 | sample_trval_tbi_ids = sample_tbi_ids[np.isin(sample_tbi_ids, test_tbi_ids, invert=True)] 20 | X_test = X_tbi[np.isin(sample_tbi_ids, test_tbi_ids)] 21 | X_trval = X_tbi[np.isin(sample_tbi_ids, test_tbi_ids, invert=True)] 22 | y_test = y_tbi[np.isin(sample_tbi_ids, test_tbi_ids)] 23 | y_trval = y_tbi[np.isin(sample_tbi_ids, test_tbi_ids, invert=True)] 24 | 25 | # Train and val split 26 | valid_tbi_ids = np.load(IDPATH+"valid_tbi_ids.npy") 27 | X_valid = X_trval[np.isin(sample_trval_tbi_ids, valid_tbi_ids)] 28 | X_train = X_trval[np.isin(sample_trval_tbi_ids, valid_tbi_ids, invert=True)] 29 | y_valid = y_trval[np.isin(sample_trval_tbi_ids, valid_tbi_ids)] 30 | y_train = y_trval[np.isin(sample_trval_tbi_ids, valid_tbi_ids, invert=True)] 31 | 32 | # Reshape 33 | if flatten: 34 | X_test = X_test.reshape(X_test.shape[0],X_test.shape[1]*X_test.shape[2]) 35 | X_valid = X_valid.reshape(X_valid.shape[0],X_valid.shape[1]*X_valid.shape[2]) 36 | X_train = X_train.reshape(X_train.shape[0],X_train.shape[1]*X_train.shape[2]) 37 | else: # Reshape for LSTM (nsamples, time steps, nfeats) 38 | X_test = np.swapaxes(X_test,1,2) 39 | X_valid = np.swapaxes(X_valid,1,2) 40 | X_train = np.swapaxes(X_train,1,2) 41 | 42 | return(X_test, y_test, X_valid, y_valid, X_train, y_train) 43 | 44 | 45 | #################################### 46 | ############ XGB CODE ############## 47 | #################################### 48 | 49 | def train_save_xgb_model(X_train,y_train,X_valid,y_valid, 50 | RESDIR,param,hosp_data,data_type): 51 | # Convert to xgb format 52 | dtrain = xgb.DMatrix(X_train, label=y_train) 53 | dvalid = xgb.DMatrix(X_valid, label=y_valid) 54 | 55 | # Train Model and save it 56 | save_path = RESDIR+"hosp{}_data/{}/".format(hosp_data,data_type) 57 | if not os.path.exists(save_path): os.makedirs(save_path) 58 | evallist = [(dvalid,"eval")] 59 | 60 | old_stdout = sys.stdout 61 | sys.stdout = open(save_path+'history.txt', 'w') 62 | bst = xgb.train(param, dtrain, 2000, evallist, early_stopping_rounds=5) 63 | sys.stdout = old_stdout 64 | bst.save_model(save_path+'mod_eta{}.model'.format(param['eta'])) 65 | 66 | def test_xgb_model(X_test,y_test,RESDIR,param,hosp_data,data_type): 67 | save_path = RESDIR+"hosp{}_data/{}/".format(hosp_data,data_type) 68 | if DEBUG: print("[DEBUG] Loading model from {}".format(save_path)) 69 | bst = xgb.Booster() 70 | bst.load_model(save_path+'mod_eta{}.model'.format(param['eta'])) 71 | dtest = xgb.DMatrix(X_test) 72 | ypred = bst.predict(dtest) 73 | np.save(save_path+"ypred.npy",ypred) 74 | np.save(save_path+"y_test.npy",y_test) 75 | auc = metrics.average_precision_score(y_test, ypred) 76 | np.random.seed(231) 77 | auc_lst = [] 78 | roc_auc_lst = [] 79 | for i in range(0,100): 80 | inds = np.random.choice(X_test.shape[0], X_test.shape[0], replace=True) 81 | auc = metrics.average_precision_score(y_test[inds], ypred[inds]) 82 | auc_lst.append(auc) 83 | roc_auc = metrics.roc_auc_score(y_test[inds], ypred[inds]) 84 | roc_auc_lst.append(roc_auc) 85 | auc_lst = np.array(auc_lst) 86 | roc_auc_lst = np.array(roc_auc_lst) 87 | if DEBUG: print("[DEBUG] auc_lst.mean(): {}".format(auc_lst.mean())) 88 | if DEBUG: print("[DEBUG] roc_auc_lst.mean(): {}".format(roc_auc_lst.mean())) 89 | 90 | SP = RESDIR+"hosp{}_data/".format(hosp_data) 91 | f = open('{}conf_int_hospdata{}_prauc.txt'.format(SP,hosp_data),'a') 92 | f.write("{}, {}+-{}\n".format(data_type,auc_lst.mean().round(4),2*np.std(auc_lst).round(4))) 93 | f.close() 94 | f = open('{}conf_int_hospdata{}_rocauc.txt'.format(SP,hosp_data),'a') 95 | f.write("{}, {}+-{}\n".format(data_type,roc_auc_lst.mean().round(4),2*np.std(roc_auc_lst).round(4))) 96 | f.close() 97 | np.save("{}auc_lst".format(save_path,data_type), auc_lst) 98 | np.save("{}roc_auc_lst".format(save_path,data_type), roc_auc_lst) 99 | 100 | 101 | #################################### 102 | ############ LSTM CODE ############# 103 | #################################### 104 | 105 | def load_min_model_helper(MPATH): 106 | print("[PROGRESS] Starting load_min_model_helper()") 107 | print("[DEBUG] MPATH {}".format(MPATH)) 108 | mfiles = os.listdir(MPATH) 109 | full_mod_name = MPATH.split("/")[-1] 110 | mfiles = [f for f in mfiles if "val_loss" in f] 111 | loss_lst = [float(f.split("val_loss:")[1].split("_")[0]) for f in mfiles] 112 | min_ind = loss_lst.index(min(loss_lst)) 113 | min_mod_name = "{}/{}".format(MPATH,mfiles[min_ind]) 114 | print("[DEBUG] min_mod_name {}".format(mfiles[min_ind])) 115 | return(load_model(min_mod_name)) 116 | 117 | def create_train_model(opt_name="rmsprop",lr=0.001,drop=0.5, 118 | b_size=1000,epoch_num=50,is_tune=False, 119 | nodesize=200): 120 | # Fixed hyperpara 121 | print("[PROGRESS] Starting create_train_model()") 122 | h1 = nodesize 123 | h2 = nodesize 124 | lookback = 60 125 | loss_func = "binary_crossentropy" 126 | 127 | # Set opt based on opt_name 128 | if opt_name is "rmsprop": 129 | opt = keras.optimizers.RMSprop(lr) 130 | elif opt_name is "sgd": 131 | opt = keras.optimizers.SGD(lr) 132 | elif opt_name is "adam": 133 | opt = keras.optimizers.Adam(lr) 134 | 135 | # Form the model name (for saving the model) 136 | mod_name = "multivariate_biglstmdropoutv3_{}n_{}n_{}ep".format(h1,h2,epoch_num) 137 | mod_name += "_{}opt_{}lr".format(opt_name,lr) 138 | mod_name += "_{}drop_{}bs".format(drop,b_size) 139 | if ISTUNE: 140 | MODDIR = PATH+"models/tune_biglstm/"+mod_name+"/" 141 | else: 142 | MODDIR = PATH+"models/"+mod_name+"/" 143 | if is_tune: MODDIR = PATH+"models/tune_biglstm/"+mod_name+"/" 144 | if not os.path.exists(MODDIR): 145 | os.makedirs(MODDIR) 146 | with open(MODDIR+"loss.txt", "w") as f: 147 | f.write("%s\t%s\t%s\t%s\n" % ("i", "train_loss", "val_loss", "epoch_time")) 148 | 149 | ########## Form Model/Data ######### 150 | X_train_lst = []; X_valid_lst = []; X_test_lst = [] 151 | sig_lst = []; encoded_lst = [] 152 | 153 | for i in range(0,len(feat_lst2)): 154 | X_train_lst.append(X_train[:,:,i:(i+1)]) 155 | X_valid_lst.append(X_valid[:,:,i:(i+1)]) 156 | X_test_lst.append(X_test[:,:,i:(i+1)]) 157 | 158 | sig = Input(shape=(lookback,1)) 159 | lstm1 = LSTM(h1, recurrent_dropout=drop, return_sequences=True) 160 | lstm2 = LSTM(h2, recurrent_dropout=drop, dropout=drop) 161 | 162 | encoded = lstm2(lstm1(sig)) 163 | sig_lst.append(sig); encoded_lst.append(encoded) 164 | 165 | merged_vector = keras.layers.concatenate(encoded_lst, axis=-1) 166 | predictions = Dense(1, activation='sigmoid')(merged_vector) 167 | 168 | model = Model(inputs=sig_lst, outputs=predictions) 169 | model = multi_gpu_model(model,gpus=GPUNUM) 170 | model.compile(optimizer=opt, loss=loss_func) 171 | 172 | ########## Training ######### 173 | print("[PROGRESS] Starting train_model()") 174 | 175 | # Train and Save 176 | start_time = time.time() 177 | for i in range(0,epoch_num): 178 | history = model.fit(X_train_lst, y_train, epochs=1, batch_size=b_size, 179 | validation_data=(X_valid_lst,y_valid)) 180 | 181 | # Save details about training 182 | train_loss = history.history['loss'][0] 183 | val_loss = history.history['val_loss'][0] 184 | epoch_time = time.time() - start_time 185 | with open(MODDIR+"loss.txt", "a") as f: 186 | f.write("%d\t%f\t%f\t%f\n" % (i, train_loss, val_loss, epoch_time)) 187 | 188 | # Save model each iteration 189 | model.save("{}val_loss:{}_epoch:{}_{}.h5".format(MODDIR,val_loss,i,mod_name)) 190 | 191 | # Return the best validation performance (if hyperpara tuning) 192 | min_mod_name = [f for f in sorted(os.listdir(MODDIR)) if "val_loss" in f][0] 193 | best_val = float(min_mod_name.split("val_loss:")[1].split("_")[0]) 194 | return(best_val) 195 | 196 | def eval_test_model(opt_name="rmsprop",lr=0.001,drop=0.5, 197 | b_size=1000,epoch_num=50,nodesize=200): 198 | 199 | # Fixed hyperpara 200 | print("[PROGRESS] Starting create_train_model()") 201 | lookback = 60; h1 = 200; h2 = 200 202 | loss_func = "binary_crossentropy" 203 | 204 | # Form the model name (for saving the model) 205 | mod_name = "multivariate_biglstmdropoutv3_{}n_{}n_{}ep".format(h1,h2,epoch_num) 206 | mod_name += "_{}opt_{}lr".format(opt_name,lr) 207 | mod_name += "_{}drop_{}bs".format(drop,b_size) 208 | MODDIR = PATH+"models/"+mod_name+"/" 209 | 210 | ########## Form Data ######### 211 | X_test_lst = [] 212 | 213 | for i in range(0,len(feat_lst2)): 214 | X_test_lst.append(X_test[:,:,i:(i+1)]) 215 | 216 | ########## Testing (optional) ######### 217 | model = load_min_model_helper(MODDIR) 218 | model = multi_gpu_model(model,gpus=GPUNUM) 219 | hosp_data = "tbi" 220 | save_path = RESDIR+"hosp{}_data/{}/".format(hosp_data,data_type) 221 | if not os.path.exists(save_path): os.makedirs(save_path) 222 | ypred = model.predict(X_test_lst) 223 | np.save(save_path+"ypred.npy",ypred) 224 | np.save(save_path+"y_test.npy",y_test) 225 | auc = metrics.average_precision_score(y_test, ypred) 226 | np.random.seed(231) 227 | auc_lst = [] 228 | roc_auc_lst = [] 229 | 230 | # Randomly sample the test set 231 | for i in range(0,100): 232 | inds = np.random.choice(X_test.shape[0], X_test.shape[0], replace=True) 233 | auc = metrics.average_precision_score(y_test[inds], ypred[inds]) 234 | auc_lst.append(auc) 235 | roc_auc = metrics.roc_auc_score(y_test[inds], ypred[inds]) 236 | roc_auc_lst.append(roc_auc) 237 | auc_lst = np.array(auc_lst) 238 | roc_auc_lst = np.array(roc_auc_lst) 239 | print("[DEBUG] auc_lst.mean(): {}".format(auc_lst.mean())) 240 | print("[DEBUG] roc_auc_lst.mean(): {}".format(roc_auc_lst.mean())) 241 | 242 | SP = RESDIR+"hosp{}_data/".format(hosp_data) 243 | f = open('{}conf_int_hospdata{}_prauc.txt'.format(SP,hosp_data),'a') 244 | f.write("{}, {}+-{}\n".format(data_type,auc_lst.mean().round(4),2*np.std(auc_lst).round(4))) 245 | f.close() 246 | f = open('{}conf_int_hospdata{}_rocauc.txt'.format(SP,hosp_data),'a') 247 | f.write("{}, {}+-{}\n".format(data_type,roc_auc_lst.mean().round(4),2*np.std(roc_auc_lst).round(4))) 248 | f.close() 249 | np.save("{}auc_lst".format(save_path,data_type), auc_lst) 250 | np.save("{}roc_auc_lst".format(save_path,data_type), roc_auc_lst) -------------------------------------------------------------------------------- /user_studies/sickness_scores/.psiturk_history: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/user_studies/sickness_scores/.psiturk_history -------------------------------------------------------------------------------- /user_studies/sickness_scores/Debrief.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import {Link} from 'react-router'; 3 | import MuiThemeProvider from 'material-ui/styles/MuiThemeProvider'; 4 | import RaisedButton from 'material-ui/RaisedButton'; 5 | import TextField from 'material-ui/TextField'; 6 | 7 | export default class Debrief extends React.Component { 8 | constructor() { 9 | super(); 10 | this.handleChange = this.handleChange.bind(this); 11 | this.start_time = new Date(); 12 | window.psiTurk.recordTrialData({ 13 | 'mark': "debrief_start", 14 | 'time': this.start_time.valueOf() 15 | }); 16 | } 17 | 18 | finish(answer) { 19 | window.psiTurk.recordTrialData({ 20 | 'mark': "debrief_end", 21 | 'answer': answer, 22 | 'text': this.text, 23 | 'time': new Date().valueOf(), 24 | 'response_time': new Date() - this.start_time 25 | }); 26 | 27 | psiTurk.saveData({ 28 | success: function(){ 29 | psiTurk.completeHIT(); 30 | }, 31 | error: this.prompt_resubmit 32 | }); 33 | } 34 | 35 | handleChange(event) { 36 | this.text = event.target.value; 37 | } 38 | 39 | render() { 40 | return ( 41 | 42 |
43 | 44 |

Debriefing

45 | 46 |
47 | 48 |

Thank you for your participation in our study! Your anonymous data makes an 49 | important contribution to our understanding of human machine interaction.

50 | 51 |

If you have any questions about this research, you may contact slund1@cs.washington.edu.

52 |
53 | 60 |
61 |
62 |
63 | 64 |

I feel that I have been adequately debriefed about the nature 65 | of the study. The investigator has explained the purposes of the 66 | research to me, and I feel that any questions I have asked were 67 | satisfactorily answered.

68 | 69 | 72 | 73 |
74 |
75 | this.finish("agree")} label="Yes, I agree. Complete this HIT" primary={true} />

76 | this.finish("questions")} label="No, please withhold my data. I will contact the experimenter with questions." /> 77 |
78 | 79 |
80 |
81 | ); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /user_studies/sickness_scores/Instructions.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import {hashHistory} from 'react-router'; 3 | import RaisedButton from 'material-ui/RaisedButton'; 4 | 5 | export default class Instructions extends React.Component { 6 | constructor() { 7 | super(); 8 | this.next_page = this.next_page.bind(this); 9 | 10 | this.start_time = new Date().getTime(); 11 | window.psiTurk.recordTrialData({ 12 | 'mark': "instructions_start", 13 | 'condition': condition, 14 | 'time': this.start_time.valueOf() 15 | }); 16 | } 17 | 18 | next_page() { 19 | window.psiTurk.recordTrialData({ 20 | 'mark': "instructions_stop", 21 | 'time': new Date().valueOf(), 22 | 'condition': condition, 23 | 'response_time': new Date() - this.start_time 24 | }); 25 | 26 | hashHistory.push("/questions"); 27 | // if (condition === 0) hashHistory.push("/allocate1"); 28 | // else if (condition === 1) hashHistory.push("/allocate2"); 29 | } 30 | 31 | render() { 32 | return ( 33 |
34 |

Instructions

35 |
36 | 37 |
38 | You will be asked to allocate blame for a person's "sickness score", 39 | where 0 is healthy and higher numbers mean more sickness. There are 40 | 4 different ways to measure sickness (A, B, C and D). There 41 | are also 3 different people for each score. This leads to 12 42 | total questions. Each question should not take very long to 43 | complete, the goal is simply for you to assign blame among a person's 44 | symptoms in a way you consider the most intuitive. 45 |
46 | 47 |
48 | 49 |
50 |
51 | ); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /user_studies/sickness_scores/Main.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import {Router, Route, hashHistory} from 'react-router' 3 | import MuiThemeProvider from 'material-ui/styles/MuiThemeProvider' 4 | import Instructions from './Instructions' 5 | import Questions from './Questions' 6 | import Debrief from './Debrief' 7 | 8 | export default class Main extends React.Component { 9 | constructor() { 10 | super(); 11 | } 12 | 13 | render() { 14 | return ( 15 | 16 |
17 | 18 | 19 | 20 | 21 | 22 |
23 |
24 | ); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /user_studies/sickness_scores/Questions.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import {Link, hashHistory} from 'react-router'; 3 | import MuiThemeProvider from 'material-ui/styles/MuiThemeProvider'; 4 | import RaisedButton from 'material-ui/RaisedButton'; 5 | import TextField from 'material-ui/TextField'; 6 | 7 | export default class QuestionA1 extends React.Component { 8 | constructor() { 9 | super(); 10 | this.state = { 11 | fever: 0, 12 | cough: 0, 13 | headache: 0, 14 | person: 0, 15 | score: 0 16 | }; 17 | this.last_time = new Date().getTime(); 18 | this.handleChange = this.handleChange.bind(this); 19 | 20 | this.symptoms = [ 21 | { 22 | 'fever': true, 23 | 'cough': true, 24 | 'headache': true 25 | }, 26 | { 27 | 'fever': false, 28 | 'cough': true, 29 | 'headache': true 30 | }, 31 | { 32 | 'fever': false, 33 | 'cough': false, 34 | 'headache': true 35 | } 36 | ] 37 | 38 | this.score_names = ["A", "B", "C", "D"]; 39 | this.score_methods = [ 40 | { // XOR 41 | "true true": 4, 42 | "false true": 8, 43 | "false false": 0 44 | }, 45 | { // AND 46 | "true true": 10, 47 | "false true": 2, 48 | "false false": 0 49 | }, 50 | { // OR 51 | "true true": 10, 52 | "false true": 8, 53 | "false false": 0 54 | }, 55 | { // SUM 56 | "true true": 4, 57 | "false true": 2, 58 | "false false": 0 59 | } 60 | ]; 61 | 62 | 63 | window.psiTurk.recordTrialData({ 64 | 'mark': "question"+this.score_names[this.state.score]+this.person+"_start", 65 | 'time': this.last_time.valueOf() 66 | }); 67 | } 68 | 69 | handleChange(event) { 70 | if (event.target.value !== undefined) { 71 | var data = {}; 72 | data[event.target.name] = event.target.value; 73 | this.setState(data); 74 | } 75 | } 76 | 77 | saveAnswers() { 78 | if (this.state.person != 2 && parseInt(this.state.fever) + parseInt(this.state.cough) + parseInt(this.state.headache) === 0) { 79 | alert("Please consider carefully how to assign blame among the symptoms before submitting."); 80 | } else { 81 | window.psiTurk.recordTrialData({ 82 | 'mark': "question"+this.score_names[this.state.score]+this.state.person+"_stop", 83 | 'fever': this.symptoms[this.state.person]['fever'], 84 | 'cough': this.symptoms[this.state.person]['cough'], 85 | 'headache': this.symptoms[this.state.person]['headache'], 86 | 'fever_credit': parseInt(this.state.fever), 87 | 'cough_credit': parseInt(this.state.cough), 88 | 'headache_credit': parseInt(this.state.headache), 89 | 'time': new Date().valueOf(), 90 | 'condition': condition, 91 | 'response_time': new Date().getTime() - this.last_time 92 | }); 93 | 94 | this.state.fever = 0; 95 | this.state.cough = 0; 96 | this.state.headache = 0; 97 | 98 | if (this.state.person < 2) { 99 | this.state.person++; 100 | } else if (this.state.score < 3) { 101 | this.state.person = 0; 102 | this.state.score++; 103 | } else { 104 | hashHistory.push("/debrief"); 105 | } 106 | this.setState(this.state); 107 | this.last_time = new Date().getTime(); 108 | } 109 | } 110 | 111 | render() { 112 | 113 | var fever = this.symptoms[this.state.person]['fever']; 114 | var cough = this.symptoms[this.state.person]['cough']; 115 | 116 | var interaction_effect_rule; 117 | var interaction_effect_result; 118 | if (this.state.score == 0) { 119 | interaction_effect_rule = ( 120 | 121 | 122 | If they have a fever OR a cough but not both: 123 | 124 | 125 | +6 points 126 | 127 | 128 | ); 129 | 130 | if ((fever || cough) && !(fever && cough)) { 131 | interaction_effect_result = ( 132 | 133 | 134 | They have a fever OR a cough but not both: 135 | 136 | 137 | +6 points 138 | 139 | 140 | ); 141 | } 142 | } else if (this.state.score == 1) { 143 | interaction_effect_rule = ( 144 | 145 | 146 | If they have a fever AND a cough: 147 | 148 | 149 | +6 points 150 | 151 | 152 | ); 153 | 154 | if (fever && cough) { 155 | interaction_effect_result = ( 156 | 157 | 158 | They have a fever AND a cough: 159 | 160 | 161 | +6 points 162 | 163 | 164 | ); 165 | } 166 | } else if (this.state.score == 2) { 167 | interaction_effect_rule = ( 168 | 169 | 170 | If they have a fever OR a cough: 171 | 172 | 173 | +6 points 174 | 175 | 176 | ); 177 | 178 | if (fever || cough) { 179 | interaction_effect_result = ( 180 | 181 | 182 | They have a fever OR a cough: 183 | 184 | 185 | +6 points 186 | 187 | 188 | ); 189 | } 190 | } 191 | 192 | var fever_effect_result; 193 | if (fever) { 194 | fever_effect_result = ( 195 | 196 | 197 | They have a fever: 198 | 199 | 200 | +2 points 201 | 202 | 203 | ); 204 | } 205 | var cough_effect_result; 206 | if (cough) { 207 | cough_effect_result = ( 208 | 209 | 210 | They have a cough: 211 | 212 | 213 | +2 points 214 | 215 | 216 | ); 217 | } 218 | 219 | if (!fever && !cough) { 220 | fever_effect_result = ( 221 | 222 | 223 | They have neither a fever nor a cough. 224 | 225 | 226 | 227 | 228 | 229 | ); 230 | } 231 | 232 | var total_score = this.score_methods[this.state.score][fever + " " + cough]; 233 | 234 | return ( 235 | 236 |
237 | 238 |

Assign blame among symptoms (score {this.score_names[this.state.score]}, person {this.state.person+1})

239 | 240 |
241 | 242 |
243 | 244 |

"Sickness score {this.score_names[this.state.score]}" is computed in the following manner:

245 | 246 |
247 | 248 | 249 | 252 | 255 | 256 | 257 | 260 | 263 | 264 | {interaction_effect_rule} 265 |
250 | If they have a fever: 251 | 253 | +2 points 254 |
258 | If they have a cough: 259 | 261 | +2 points 262 |
266 |
267 |
268 | Person {this.state.person+1} has the following symptoms:
269 |
270 | 271 | 272 | 275 | 278 | 279 | 280 | 283 | 286 | 287 | 288 | 291 | 294 | 295 |
273 | Fever: 274 | 276 | {this.symptoms[this.state.person]['fever'] ? 'YES' : 'NO'} 277 |
281 | Cough: 282 | 284 | {this.symptoms[this.state.person]['cough'] ? 'YES' : 'NO'} 285 |
289 | Headache: 290 | 292 | {this.symptoms[this.state.person]['headache'] ? 'YES' : 'NO'} 293 |
296 |
297 | This leads to a total sickness score {this.score_names[this.state.score]} value of {total_score} points because:
298 |
299 | 300 | {fever_effect_result} 301 | {cough_effect_result} 302 | {interaction_effect_result} 303 |
304 |
305 |
306 | Using numbers please assign blame for their sickness score of {total_score} points among the following symptoms{this.state.person == 2 ? ' (zero blame is okay)' : ''}:
307 | 308 |
309 |
310 | 311 | 312 | 316 | 320 | 324 | 325 |
313 | point(s)
314 | Fever: {this.symptoms[this.state.person]['fever'] ? 'YES' : 'NO'} 315 |
317 | point(s)
318 | Cough: {this.symptoms[this.state.person]['cough'] ? 'YES' : 'NO'} 319 |
321 | point(s)
322 | Headache: {this.symptoms[this.state.person]['headache'] ? 'YES' : 'NO'} 323 |
326 |
327 | 328 |
329 |
330 | this.saveAnswers()} /> 331 |
332 | 333 |
334 |
335 | ); 336 | } 337 | } 338 | -------------------------------------------------------------------------------- /user_studies/sickness_scores/README.md: -------------------------------------------------------------------------------- 1 | # Exclusive OR function experiment based on sickness score 2 | 3 | ```shell 4 | npm install 5 | npm build 6 | ``` 7 | 8 | Using a python 2 environment 9 | 10 | ``` 11 | pip install psiturk 12 | ``` 13 | 14 | Create a `~/.psiturkconfig` with your credentials: 15 | 16 | ``` 17 | [AWS Access] 18 | aws_access_key_id = XXXXXXXXXXXXXX 19 | aws_secret_access_key = XXXXXXXXXXXXXX 20 | aws_region = us-east-1 21 | 22 | [psiTurk Access] 23 | psiturk_access_key_id = XXXXXXXXXXXXXX 24 | psiturk_secret_access_id = XXXXXXXXXXXXXX 25 | ``` 26 | 27 | In `config.txt` replace the host name with the public name of the machine. 28 | 29 | In the current directory run PsiTurk 30 | 31 | ``` 32 | psiturk 33 | 34 | > server on 35 | > debug 36 | ``` 37 | 38 | Now you can test things and download results with 39 | ``` 40 | > download_datafiles 41 | ``` 42 | -------------------------------------------------------------------------------- /user_studies/sickness_scores/config.txt: -------------------------------------------------------------------------------- 1 | [HIT Configuration] 2 | title = Intuitive credit allocation experiment 3 | description = Allocate blame for sickness among three symptoms in a simple scenario. 4 | amt_keywords = explainable, interpretable, machine learning 5 | lifetime = 24 6 | us_only = true 7 | approve_requirement = 95 8 | contact_email_on_error = slund1@cs.washington.edu 9 | ad_group = Explainable testing 10 | psiturk_keywords = shap 11 | organization_name = University of Washington 12 | browser_exclude_rule = MSIE, mobile, tablet 13 | 14 | [Database Parameters] 15 | database_url = sqlite:///participants.db 16 | table_name = turkdemo 17 | 18 | [Server Parameters] 19 | host = 0.0.0.0 20 | port = 22363 21 | cutoff_time = 30 22 | logfile = server.log 23 | loglevel = 2 24 | debug = true 25 | login_username = examplename 26 | login_pw = examplepassword221s0982927998292900 27 | threads = auto 28 | secret_key = 'ksd9f98239s9df8skjkj2j32k09a0-k2lk32kl' 29 | #certfile = 30 | #keyfile = 31 | 32 | [Task Parameters] 33 | experiment_code_version = 1.0 34 | num_conds = 1 35 | num_counters = 1 36 | 37 | [Shell Parameters] 38 | launch_in_sandbox_mode = true 39 | 40 | # If you are not using the psiturk ad server, set `use_psiturk_ad_server` to `false` and point `ad_location` to your proxy server and . Format the ad_location like this: 41 | # 42 | # https://:/ad 43 | 44 | use_psiturk_ad_server = true 45 | ad_location = false 46 | -------------------------------------------------------------------------------- /user_studies/sickness_scores/index.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDom from 'react-dom'; 3 | import injectTapEventPlugin from 'react-tap-event-plugin'; 4 | import Main from './Main'; 5 | 6 | // Needed for onTouchTap for material-ui 7 | // http://stackoverflow.com/a/34015469/988941 8 | injectTapEventPlugin(); 9 | 10 | // Save some globals for the inline scripts to access 11 | window.IML = { 12 | Main: Main, 13 | React: React, 14 | ReactDom: ReactDom 15 | }; 16 | -------------------------------------------------------------------------------- /user_studies/sickness_scores/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "iml-sickness-score-experiment", 3 | "version": "0.1.0", 4 | "description": "Measure how people explain a sickness score.", 5 | "main": "index.jsx", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "dev": "node_modules/.bin/webpack --content-base build/", 9 | "build": "node_modules/.bin/webpack" 10 | }, 11 | "author": "Scott Lundberg", 12 | "repository": "https://github.com/interpretable-ml/benchmark/experiments/sickness-score", 13 | "license": "MIT", 14 | "dependencies": { 15 | "d3": "^4", 16 | "lodash": "^4", 17 | "material-ui": "^0.16", 18 | "react": "^15", 19 | "react-dom": "^15", 20 | "react-router": "^3", 21 | "react-tap-event-plugin": "^2" 22 | }, 23 | "devDependencies": { 24 | "babel-core": "^6", 25 | "babel-loader": "^6", 26 | "babel-preset-es2015": "^6", 27 | "babel-preset-react": "^6", 28 | "webpack": "^2", 29 | "webpack-dev-server": "^2" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/css/style.css: -------------------------------------------------------------------------------- 1 | 2 | body { 3 | padding:0px; 4 | margin: 0px; 5 | background-color: white; 6 | color: black; 7 | font-weight: 300; 8 | font-size: 13pt; 9 | } 10 | 11 | 12 | /* text styling */ 13 | strong { 14 | font-weight: 600; 15 | } 16 | 17 | .warm { 18 | color: steelblue; 19 | text-align: justify; 20 | } 21 | 22 | .cool { 23 | color: #ccccff; 24 | } 25 | 26 | h1 { 27 | margin-top: 0px; 28 | } 29 | 30 | 31 | /* these are all the same for now, but easy to modify individually */ 32 | #container-not-an-ad, #container-error, #container-exp, #container-consent, #container-instructions, #container-questionnaire { 33 | background: white; 34 | margin: 30px; 35 | } 36 | 37 | /* ad.html - the ad that people view first */ 38 | #adlogo { 39 | width: 140px; 40 | padding: 2px; 41 | border: 1px solid #ccc; 42 | margin-right: 1em; 43 | } 44 | 45 | /* consent.html - the consent form */ 46 | #consent { 47 | margin: 0 auto; 48 | margin-top: 0; 49 | width: 800px; 50 | font-size: 100%; 51 | } 52 | 53 | .legal { 54 | text-align: justify; 55 | font-size: 11pt; 56 | line-height: 1.1em; 57 | } 58 | 59 | #trial { 60 | margin: 0 auto; 61 | width: 800px; 62 | text-align: center; 63 | } 64 | 65 | #query { 66 | color: darkgray; 67 | } 68 | 69 | .question { 70 | margin-bottom: 30px; 71 | } 72 | 73 | /* errors */ 74 | #error img { 75 | padding-right: 10px; 76 | padding-top: 15px; 77 | } 78 | -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/user_studies/sickness_scores/static/favicon.ico -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/user_studies/sickness_scores/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/user_studies/sickness_scores/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/user_studies/sickness_scores/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/images/Example1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/user_studies/sickness_scores/static/images/Example1.png -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/images/Example1_nobar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/user_studies/sickness_scores/static/images/Example1_nobar.png -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/images/Example2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/user_studies/sickness_scores/static/images/Example2.png -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/images/Example2_nobar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/user_studies/sickness_scores/static/images/Example2_nobar.png -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/images/university.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suinleelab/treeexplainer-study/82dd0fec4df709683a957babc805ff7ac8139a52/user_studies/sickness_scores/static/images/university.png -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/js/task.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Requires: 3 | * psiturk.js 4 | * utils.js 5 | */ 6 | 7 | // Initalize psiturk object 8 | var psiTurk = new PsiTurk(uniqueId, adServerLoc, mode); 9 | 10 | var mycondition = condition; // these two variables are passed by the psiturk server process 11 | var mycounterbalance = counterbalance; // they tell you which condition you have been assigned to 12 | // they are not used in the stroop code but may be useful to you 13 | 14 | // All pages to be loaded 15 | var pages = [ 16 | "instructions/instruct-1.html", 17 | "instructions/instruct-2.html", 18 | "instructions/instruct-3.html", 19 | "instructions/instruct-ready.html", 20 | "stage.html", 21 | "postquestionnaire.html" 22 | ]; 23 | 24 | psiTurk.preloadPages(pages); 25 | 26 | var instructionPages = [ // add as a list as many pages as you like 27 | "instructions/instruct-1.html", 28 | "instructions/instruct-2.html", 29 | "instructions/instruct-3.html", 30 | "instructions/instruct-ready.html" 31 | ]; 32 | 33 | 34 | /******************** 35 | * HTML manipulation 36 | * 37 | * All HTML files in the templates directory are requested 38 | * from the server when the PsiTurk object is created above. We 39 | * need code to get those pages from the PsiTurk object and 40 | * insert them into the document. 41 | * 42 | ********************/ 43 | 44 | /******************** 45 | * STROOP TEST * 46 | ********************/ 47 | var StroopExperiment = function() { 48 | 49 | var wordon, // time word is presented 50 | listening = false; 51 | 52 | // Stimuli for a basic Stroop experiment 53 | var stims = [ 54 | ["SHIP", "red", "unrelated"], 55 | ["MONKEY", "green", "unrelated"], 56 | ["ZAMBONI", "blue", "unrelated"], 57 | ["RED", "red", "congruent"], 58 | ["GREEN", "green", "congruent"], 59 | ["BLUE", "blue", "congruent"], 60 | ["GREEN", "red", "incongruent"], 61 | ["BLUE", "green", "incongruent"], 62 | ["RED", "blue", "incongruent"] 63 | ]; 64 | 65 | stims = _.shuffle(stims); 66 | 67 | var next = function() { 68 | if (stims.length===0) { 69 | finish(); 70 | } 71 | else { 72 | stim = stims.shift(); 73 | show_word( stim[0], stim[1] ); 74 | wordon = new Date().getTime(); 75 | listening = true; 76 | d3.select("#query").html('

Type "R" for Red, "B" for blue, "G" for green.

'); 77 | } 78 | }; 79 | 80 | var response_handler = function(e) { 81 | if (!listening) return; 82 | 83 | var keyCode = e.keyCode, 84 | response; 85 | 86 | switch (keyCode) { 87 | case 82: 88 | // "R" 89 | response="red"; 90 | break; 91 | case 71: 92 | // "G" 93 | response="green"; 94 | break; 95 | case 66: 96 | // "B" 97 | response="blue"; 98 | break; 99 | default: 100 | response = ""; 101 | break; 102 | } 103 | if (response.length>0) { 104 | listening = false; 105 | var hit = response == stim[1]; 106 | var rt = new Date().getTime() - wordon; 107 | 108 | psiTurk.recordTrialData({'phase':"TEST", 109 | 'word':stim[0], 110 | 'color':stim[1], 111 | 'relation':stim[2], 112 | 'response':response, 113 | 'hit':hit, 114 | 'rt':rt} 115 | ); 116 | remove_word(); 117 | next(); 118 | } 119 | }; 120 | 121 | var finish = function() { 122 | $("body").unbind("keydown", response_handler); // Unbind keys 123 | currentview = new Questionnaire(); 124 | }; 125 | 126 | var show_word = function(text, color) { 127 | d3.select("#stim") 128 | .append("div") 129 | .attr("id","word") 130 | .style("color",color) 131 | .style("text-align","center") 132 | .style("font-size","150px") 133 | .style("font-weight","400") 134 | .style("margin","20px") 135 | .text(text); 136 | }; 137 | 138 | var remove_word = function() { 139 | d3.select("#word").remove(); 140 | }; 141 | 142 | 143 | // Load the stage.html snippet into the body of the page 144 | psiTurk.showPage('stage.html'); 145 | 146 | // Register the response handler that is defined above to handle any 147 | // key down events. 148 | $("body").focus().keydown(response_handler); 149 | 150 | // Start the test 151 | next(); 152 | }; 153 | 154 | 155 | /**************** 156 | * Questionnaire * 157 | ****************/ 158 | 159 | var Questionnaire = function() { 160 | 161 | var error_message = "

Oops!

Something went wrong submitting your HIT. This might happen if you lose your internet connection. Press the button to resubmit.

"; 162 | 163 | record_responses = function() { 164 | 165 | psiTurk.recordTrialData({'phase':'postquestionnaire', 'status':'submit'}); 166 | 167 | $('textarea').each( function(i, val) { 168 | psiTurk.recordUnstructuredData(this.id, this.value); 169 | }); 170 | $('select').each( function(i, val) { 171 | psiTurk.recordUnstructuredData(this.id, this.value); 172 | }); 173 | 174 | }; 175 | 176 | prompt_resubmit = function() { 177 | document.body.innerHTML = error_message; 178 | $("#resubmit").click(resubmit); 179 | }; 180 | 181 | resubmit = function() { 182 | document.body.innerHTML = "

Trying to resubmit...

"; 183 | reprompt = setTimeout(prompt_resubmit, 10000); 184 | 185 | psiTurk.saveData({ 186 | success: function() { 187 | clearInterval(reprompt); 188 | psiTurk.computeBonus('compute_bonus', function(){finish()}); 189 | }, 190 | error: prompt_resubmit 191 | }); 192 | }; 193 | 194 | // Load the questionnaire snippet 195 | psiTurk.showPage('postquestionnaire.html'); 196 | psiTurk.recordTrialData({'phase':'postquestionnaire', 'status':'begin'}); 197 | 198 | $("#next").click(function () { 199 | record_responses(); 200 | psiTurk.saveData({ 201 | success: function(){ 202 | psiTurk.computeBonus('compute_bonus', function() { 203 | psiTurk.completeHIT(); // when finished saving compute bonus, the quit 204 | }); 205 | }, 206 | error: prompt_resubmit}); 207 | }); 208 | 209 | 210 | }; 211 | 212 | // Task object to keep track of the current phase 213 | var currentview; 214 | 215 | /******************* 216 | * Run Task 217 | ******************/ 218 | $(window).load( function(){ 219 | psiTurk.doInstructions( 220 | instructionPages, // a list of pages you want to display in sequence 221 | function() { currentview = new StroopExperiment(); } // what you want to do when you are done with instructions 222 | ); 223 | }); 224 | -------------------------------------------------------------------------------- /user_studies/sickness_scores/static/js/utils.js: -------------------------------------------------------------------------------- 1 | 2 | function AssertException(message) { this.message = message; } 3 | AssertException.prototype.toString = function () { 4 | return 'AssertException: ' + this.message; 5 | }; 6 | 7 | function assert(exp, message) { 8 | if (!exp) { 9 | throw new AssertException(message); 10 | } 11 | } 12 | 13 | // Mean of booleans (true==1; false==0) 14 | function boolpercent(arr) { 15 | var count = 0; 16 | for (var i=0; i").attr(t);this.setElement(e,false)}else{this.setElement(h.result(this,"el"),false)}}});a.sync=function(t,e,i){var r=k[t];h.defaults(i||(i={}),{emulateHTTP:a.emulateHTTP,emulateJSON:a.emulateJSON});var s={type:r,dataType:"json"};if(!i.url){s.url=h.result(e,"url")||U()}if(i.data==null&&e&&(t==="create"||t==="update"||t==="patch")){s.contentType="application/json";s.data=JSON.stringify(i.attrs||e.toJSON(i))}if(i.emulateJSON){s.contentType="application/x-www-form-urlencoded";s.data=s.data?{model:s.data}:{}}if(i.emulateHTTP&&(r==="PUT"||r==="DELETE"||r==="PATCH")){s.type="POST";if(i.emulateJSON)s.data._method=r;var n=i.beforeSend;i.beforeSend=function(t){t.setRequestHeader("X-HTTP-Method-Override",r);if(n)return n.apply(this,arguments)}}if(s.type!=="GET"&&!i.emulateJSON){s.processData=false}if(s.type==="PATCH"&&window.ActiveXObject&&!(window.external&&window.external.msActiveXFilteringEnabled)){s.xhr=function(){return new ActiveXObject("Microsoft.XMLHTTP")}}var o=i.xhr=a.ajax(h.extend(s,i));e.trigger("request",e,o,i);return o};var k={create:"POST",update:"PUT",patch:"PATCH","delete":"DELETE",read:"GET"};a.ajax=function(){return a.$.ajax.apply(a.$,arguments)};var S=a.Router=function(t){t||(t={});if(t.routes)this.routes=t.routes;this._bindRoutes();this.initialize.apply(this,arguments)};var $=/\((.*?)\)/g;var T=/(\(\?)?:\w+/g;var H=/\*\w+/g;var A=/[\-{}\[\]+?.,\\\^$|#\s]/g;h.extend(S.prototype,o,{initialize:function(){},route:function(t,e,i){if(!h.isRegExp(t))t=this._routeToRegExp(t);if(h.isFunction(e)){i=e;e=""}if(!i)i=this[e];var r=this;a.history.route(t,function(s){var n=r._extractParameters(t,s);i&&i.apply(r,n);r.trigger.apply(r,["route:"+e].concat(n));r.trigger("route",e,n);a.history.trigger("route",r,e,n)});return this},navigate:function(t,e){a.history.navigate(t,e);return this},_bindRoutes:function(){if(!this.routes)return;this.routes=h.result(this,"routes");var t,e=h.keys(this.routes);while((t=e.pop())!=null){this.route(t,this.routes[t])}},_routeToRegExp:function(t){t=t.replace(A,"\\$&").replace($,"(?:$1)?").replace(T,function(t,e){return e?t:"([^/]+)"}).replace(H,"(.*?)");return new RegExp("^"+t+"$")},_extractParameters:function(t,e){var i=t.exec(e).slice(1);return h.map(i,function(t){return t?decodeURIComponent(t):null})}});var I=a.History=function(){this.handlers=[];h.bindAll(this,"checkUrl");if(typeof window!=="undefined"){this.location=window.location;this.history=window.history}};var N=/^[#\/]|\s+$/g;var P=/^\/+|\/+$/g;var O=/msie [\w.]+/;var C=/\/$/;I.started=false;h.extend(I.prototype,o,{interval:50,getHash:function(t){var e=(t||this).location.href.match(/#(.*)$/);return e?e[1]:""},getFragment:function(t,e){if(t==null){if(this._hasPushState||!this._wantsHashChange||e){t=this.location.pathname;var i=this.root.replace(C,"");if(!t.indexOf(i))t=t.substr(i.length)}else{t=this.getHash()}}return t.replace(N,"")},start:function(t){if(I.started)throw new Error("Backbone.history has already been started");I.started=true;this.options=h.extend({},{root:"/"},this.options,t);this.root=this.options.root;this._wantsHashChange=this.options.hashChange!==false;this._wantsPushState=!!this.options.pushState;this._hasPushState=!!(this.options.pushState&&this.history&&this.history.pushState);var e=this.getFragment();var i=document.documentMode;var r=O.exec(navigator.userAgent.toLowerCase())&&(!i||i<=7);this.root=("/"+this.root+"/").replace(P,"/");if(r&&this._wantsHashChange){this.iframe=a.$('