├── .gitignore ├── LICENSE ├── README.md ├── configs ├── ag_news │ ├── albert │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ ├── svs-3800.jsonnet │ │ └── svs-7600.jsonnet │ ├── bert │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ ├── svs-3800.jsonnet │ │ └── svs-7600.jsonnet │ └── roberta │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ ├── svs-3800.jsonnet │ │ └── svs-7600.jsonnet ├── imdb │ ├── albert │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ ├── svs-10000.jsonnet │ │ ├── svs-15000.jsonnet │ │ ├── svs-20000.jsonnet │ │ ├── svs-25000.jsonnet │ │ └── svs-5000.jsonnet │ ├── bert │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── lime_ignite.jsonnet │ │ ├── occ.jsonnet │ │ ├── svs-10800.jsonnet │ │ ├── svs-14400.jsonnet │ │ ├── svs-18000.jsonnet │ │ ├── svs-21500.jsonnet │ │ ├── svs-25000.jsonnet │ │ ├── svs-3600.jsonnet │ │ └── svs-7200.jsonnet │ ├── electra │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ ├── svs-10800.jsonnet │ │ ├── svs-14400.jsonnet │ │ ├── svs-18000.jsonnet │ │ ├── svs-21500.jsonnet │ │ ├── svs-25000.jsonnet │ │ ├── svs-3600.jsonnet │ │ └── svs-7200.jsonnet │ ├── roberta │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ ├── svs-10000.jsonnet │ │ ├── svs-15000.jsonnet │ │ ├── svs-20000.jsonnet │ │ ├── svs-25000.jsonnet │ │ └── svs-5000.jsonnet │ └── xlnet │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ ├── svs-10000.jsonnet │ │ ├── svs-15000.jsonnet │ │ ├── svs-20000.jsonnet │ │ ├── svs-25000.jsonnet │ │ └── svs-5000.jsonnet ├── mnli │ ├── albert │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ ├── svs-3250.jsonnet │ │ ├── svs-6500.jsonnet │ │ └── svs-9815.jsonnet │ ├── bert │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ └── svs.jsonnet │ ├── electra │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ └── svs.jsonnet │ ├── roberta │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ ├── svs-3250.jsonnet │ │ ├── svs-6500.jsonnet │ │ └── svs-9815.jsonnet │ └── xlnet │ │ ├── lds.jsonnet │ │ ├── lgs.jsonnet │ │ ├── lgxa.jsonnet │ │ ├── lig.jsonnet │ │ ├── lime-100.jsonnet │ │ ├── lime.jsonnet │ │ ├── occ.jsonnet │ │ ├── svs-3250.jsonnet │ │ ├── svs-6500.jsonnet │ │ └── svs-9815.jsonnet ├── qqp │ ├── albert │ │ └── lgxa.jsonnet │ └── bert │ │ └── lgxa.jsonnet └── xnli │ ├── albert │ ├── lds.jsonnet │ ├── lgs.jsonnet │ ├── lgxa.jsonnet │ ├── lig.jsonnet │ ├── lime-100.jsonnet │ ├── lime.jsonnet │ ├── occ.jsonnet │ └── svs.jsonnet │ ├── bert │ ├── lds.jsonnet │ ├── lgs.jsonnet │ ├── lgxa.jsonnet │ ├── lig.jsonnet │ ├── lime-100.jsonnet │ ├── lime.jsonnet │ ├── occ.jsonnet │ └── svs.jsonnet │ ├── electra │ ├── lds.jsonnet │ ├── lgs.jsonnet │ ├── lgxa.jsonnet │ ├── lig.jsonnet │ ├── lime-100.jsonnet │ ├── lime.jsonnet │ ├── occ.jsonnet │ └── svs.jsonnet │ ├── roberta │ ├── lds.jsonnet │ ├── lgs.jsonnet │ ├── lgxa.jsonnet │ ├── lig.jsonnet │ ├── lime-100.jsonnet │ ├── lime.jsonnet │ ├── occ.jsonnet │ └── svs.jsonnet │ └── xlnet │ ├── lds.jsonnet │ ├── lgs.jsonnet │ ├── lgxa.jsonnet │ ├── lig.jsonnet │ ├── lime-100.jsonnet │ ├── lime.jsonnet │ ├── occ.jsonnet │ └── svs.jsonnet ├── demo.ipynb ├── download_data.py ├── figures ├── imdb-429.png ├── instance-contents.png ├── logo.png ├── overview_v1.png └── spaces.png ├── requirements.txt ├── run_explainer.py ├── setup.py ├── src └── thermostat │ ├── README.md │ ├── __init__.py │ ├── data │ ├── __init__.py │ ├── additional_configs.py │ ├── dataset_utils.py │ ├── readers.py │ ├── thermostat_configs.py │ └── tokenization.py │ ├── dataset.py │ ├── explain.py │ ├── explainers │ ├── __init__.py │ ├── grad.py │ ├── iba.py │ ├── lime.py │ ├── occlusion.py │ ├── shap.py │ └── svs.py │ ├── streamlit │ ├── README.md │ ├── download.py │ ├── logo.png │ ├── run.py │ └── streamlit-menu.png │ ├── utils.py │ └── visualize.py ├── uc_expl-comp.py ├── uc_model-comp.py └── uc_rank-corr.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Locked files 2 | *.lock 3 | !dvc.lock 4 | 5 | # Compiled python modules. 6 | *.pyc 7 | 8 | # Byte-compiled 9 | _pycache__/ 10 | .cache/ 11 | 12 | # Python egg metadata, regenerated from source files by setuptools. 13 | *.egg-info 14 | .eggs/ 15 | 16 | # PyPI distribution artifacts. 17 | build/ 18 | dist/ 19 | 20 | # Environments 21 | .env 22 | .venv 23 | env/ 24 | venv/ 25 | ENV/ 26 | env.bak/ 27 | venv.bak/ 28 | 29 | # pyenv 30 | .python-version 31 | 32 | # Tests 33 | .pytest_cache/ 34 | 35 | # Other 36 | *.DS_Store 37 | 38 | # PyCharm/vscode 39 | .idea 40 | .vscode 41 | 42 | # Sphinx documentation 43 | docs/_build/ 44 | docs/source/_build/ 45 | 46 | maps/ 47 | 48 | .ipynb_checkpoints/ 49 | 50 | align_maps.py 51 | pypi-test-api-token.txt 52 | run_visualization.py 53 | runtimes.md 54 | src/thermostat/data/stats.py 55 | st-venv/ 56 | figures/ 57 | -------------------------------------------------------------------------------- /configs/ag_news/albert/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerDeepLiftShap", 13 | }, 14 | "model": { 15 | "name": "textattack/albert-base-v2-ag-news", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/ag_news/albert/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientShap", 13 | "n_samples": 5, 14 | }, 15 | "model": { 16 | "name": "textattack/albert-base-v2-ag-news", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/ag_news/albert/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientXActivation", 13 | }, 14 | "model": { 15 | "name": "textattack/albert-base-v2-ag-news", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/ag_news/albert/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerIntegratedGradients", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | }, 16 | "model": { 17 | "name": "textattack/albert-base-v2-ag-news", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/ag_news/albert/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 100, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/albert-base-v2-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/ag_news/albert/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/albert-base-v2-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/ag_news/albert/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "Occlusion", 13 | "internal_batch_size": 1, 14 | "sliding_window_shapes": [3], 15 | }, 16 | "model": { 17 | "name": "textattack/albert-base-v2-ag-news", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/ag_news/albert/svs-3800.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "end": 3800, 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/albert-base-v2-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/ag_news/albert/svs-7600.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "start": 3800, 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/albert-base-v2-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/ag_news/bert/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerDeepLiftShap", 13 | }, 14 | "model": { 15 | "name": "textattack/bert-base-uncased-ag-news", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/ag_news/bert/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientShap", 13 | "n_samples": 5, 14 | }, 15 | "model": { 16 | "name": "textattack/bert-base-uncased-ag-news", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/ag_news/bert/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientXActivation", 13 | }, 14 | "model": { 15 | "name": "textattack/bert-base-uncased-ag-news", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/ag_news/bert/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerIntegratedGradients", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | }, 16 | "model": { 17 | "name": "textattack/bert-base-uncased-ag-news", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/ag_news/bert/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 100, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/ag_news/bert/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/ag_news/bert/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "Occlusion", 13 | "internal_batch_size": 1, 14 | "sliding_window_shapes": [3], 15 | }, 16 | "model": { 17 | "name": "textattack/bert-base-uncased-ag-news", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/ag_news/bert/svs-3800.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "end": 3800, 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/ag_news/bert/svs-7600.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "start": 3800, 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/ag_news/roberta/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerDeepLiftShap", 13 | }, 14 | "model": { 15 | "name": "textattack/roberta-base-ag-news", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/ag_news/roberta/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientShap", 13 | "n_samples": 5, 14 | }, 15 | "model": { 16 | "name": "textattack/roberta-base-ag-news", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/ag_news/roberta/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientXActivation", 13 | }, 14 | "model": { 15 | "name": "textattack/roberta-base-ag-news", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/ag_news/roberta/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerIntegratedGradients", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | }, 16 | "model": { 17 | "name": "textattack/roberta-base-ag-news", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/ag_news/roberta/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 100, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/roberta-base-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/ag_news/roberta/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/roberta-base-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/ag_news/roberta/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "Occlusion", 13 | "internal_batch_size": 1, 14 | "sliding_window_shapes": [3], 15 | }, 16 | "model": { 17 | "name": "textattack/roberta-base-ag-news", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/ag_news/roberta/svs-3800.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "end": 3800, 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/roberta-base-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/ag_news/roberta/svs-7600.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "ag_news", 6 | "split": "test", 7 | "start": 3800, 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/roberta-base-ag-news", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/albert/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerDeepLiftShap", 13 | }, 14 | "model": { 15 | "name": "textattack/albert-base-v2-imdb", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/imdb/albert/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientShap", 13 | "n_samples": 5, 14 | }, 15 | "model": { 16 | "name": "textattack/albert-base-v2-imdb", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/imdb/albert/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientXActivation", 13 | }, 14 | "model": { 15 | "name": "textattack/albert-base-v2-imdb", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } -------------------------------------------------------------------------------- /configs/imdb/albert/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerIntegratedGradients", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | }, 16 | "model": { 17 | "name": "textattack/albert-base-v2-imdb", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/imdb/albert/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 100, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/albert-base-v2-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/albert/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/albert-base-v2-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/albert/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "Occlusion", 13 | "internal_batch_size": 1, 14 | "sliding_window_shapes": [3], 15 | }, 16 | "model": { 17 | "name": "textattack/albert-base-v2-imdb", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/imdb/albert/svs-10000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 5000, 8 | "end": 10000, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | "early_stopping": -1, 18 | }, 19 | "model": { 20 | "name": "textattack/albert-base-v2-imdb", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "labels"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/imdb/albert/svs-15000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 10000, 8 | "end": 15000, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | "early_stopping": -1, 18 | }, 19 | "model": { 20 | "name": "textattack/albert-base-v2-imdb", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "labels"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/imdb/albert/svs-20000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 15000, 8 | "end": 20000, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | "early_stopping": -1, 18 | }, 19 | "model": { 20 | "name": "textattack/albert-base-v2-imdb", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "labels"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/imdb/albert/svs-25000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 20000, 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | "early_stopping": -1, 17 | }, 18 | "model": { 19 | "name": "textattack/albert-base-v2-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/albert/svs-5000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "end": 5000, 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | "early_stopping": -1, 17 | }, 18 | "model": { 19 | "name": "textattack/albert-base-v2-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/bert/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerDeepLiftShap", 13 | }, 14 | "model": { 15 | "name": "textattack/bert-base-uncased-imdb", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/imdb/bert/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientShap", 13 | "n_samples": 5, 14 | }, 15 | "model": { 16 | "name": "textattack/bert-base-uncased-imdb", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/imdb/bert/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientXActivation", 13 | }, 14 | "model": { 15 | "name": "textattack/bert-base-uncased-imdb", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } -------------------------------------------------------------------------------- /configs/imdb/bert/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerIntegratedGradients", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | }, 16 | "model": { 17 | "name": "textattack/bert-base-uncased-imdb", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/imdb/bert/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 100, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/bert/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/bert/lime_ignite.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "bert-base-cased", 19 | "mode_load": "ignite", 20 | "path_model": "$HOME/models/thermostat/2021-03-08-experiment-imdb/models/2021-03-08-16-13-44.bert-base-cased.huggingface.imdb_model_f1=0.9324.pt", 21 | "tokenizer": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } -------------------------------------------------------------------------------- /configs/imdb/bert/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "Occlusion", 13 | "internal_batch_size": 1, 14 | "sliding_window_shapes": [3], 15 | }, 16 | "model": { 17 | "name": "textattack/bert-base-uncased-imdb", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/imdb/bert/svs-10800.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 7200, 8 | "end": 10800, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/bert-base-uncased-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/bert/svs-14400.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 10800, 8 | "end": 14400, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/bert-base-uncased-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/bert/svs-18000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 14400, 8 | "end": 18000, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/bert-base-uncased-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/bert/svs-21500.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 18000, 8 | "end": 21500, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/bert-base-uncased-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/bert/svs-25000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 21500, 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/bert/svs-3600.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "end": 3600, 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/bert/svs-7200.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 3600, 8 | "end": 7200, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/bert-base-uncased-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/electra/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerDeepLiftShap", 13 | }, 14 | "model": { 15 | "name": "monologg/electra-small-finetuned-imdb", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/imdb/electra/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientShap", 13 | "n_samples": 5, 14 | }, 15 | "model": { 16 | "name": "monologg/electra-small-finetuned-imdb", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/imdb/electra/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientXActivation", 13 | }, 14 | "model": { 15 | "name": "monologg/electra-small-finetuned-imdb", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/imdb/electra/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerIntegratedGradients", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | }, 16 | "model": { 17 | "name": "monologg/electra-small-finetuned-imdb", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/imdb/electra/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 100, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "monologg/electra-small-finetuned-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/electra/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "monologg/electra-small-finetuned-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/electra/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "Occlusion", 13 | "internal_batch_size": 1, 14 | "sliding_window_shapes": [3], 15 | }, 16 | "model": { 17 | "name": "monologg/electra-small-finetuned-imdb", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } -------------------------------------------------------------------------------- /configs/imdb/electra/svs-10800.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 7200, 8 | "end": 10800, 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "monologg/electra-small-finetuned-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/electra/svs-14400.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 10800, 8 | "end": 14400, 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "monologg/electra-small-finetuned-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/electra/svs-18000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 14400, 8 | "end": 18000, 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "monologg/electra-small-finetuned-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/electra/svs-21500.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 18000, 8 | "end": 21500, 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "monologg/electra-small-finetuned-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/electra/svs-25000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 21500, 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "monologg/electra-small-finetuned-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/electra/svs-3600.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "end": 3600, 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "monologg/electra-small-finetuned-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/electra/svs-7200.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 3600, 8 | "end": 7200, 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "monologg/electra-small-finetuned-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/roberta/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'special_tokens_mask', 'attention_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerDeepLiftShap", 13 | }, 14 | "model": { 15 | "name": "textattack/roberta-base-imdb", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/imdb/roberta/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'special_tokens_mask', 'attention_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientShap", 13 | "n_samples": 5, 14 | }, 15 | "model": { 16 | "name": "textattack/roberta-base-imdb", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/imdb/roberta/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'special_tokens_mask', 'attention_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientXActivation", 13 | }, 14 | "model": { 15 | "name": "textattack/roberta-base-imdb", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/imdb/roberta/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'special_tokens_mask', 'attention_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerIntegratedGradients", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | }, 16 | "model": { 17 | "name": "textattack/roberta-base-imdb", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/imdb/roberta/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 100, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/roberta-base-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/roberta/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/roberta-base-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/roberta/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "Occlusion", 13 | "internal_batch_size": 1, 14 | "sliding_window_shapes": [3], 15 | }, 16 | "model": { 17 | "name": "textattack/roberta-base-imdb", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/imdb/roberta/svs-10000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 5000, 8 | "end": 10000, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 2, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 2, 16 | "n_samples": 25, 17 | "early_stopping": -1, 18 | }, 19 | "model": { 20 | "name": "textattack/roberta-base-imdb", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "labels"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/imdb/roberta/svs-15000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 10000, 8 | "end": 15000, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 2, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 2, 16 | "n_samples": 25, 17 | "early_stopping": -1, 18 | }, 19 | "model": { 20 | "name": "textattack/roberta-base-imdb", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "labels"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/imdb/roberta/svs-20000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 15000, 8 | "end": 20000, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 2, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 2, 16 | "n_samples": 25, 17 | "early_stopping": -1, 18 | }, 19 | "model": { 20 | "name": "textattack/roberta-base-imdb", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "labels"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/imdb/roberta/svs-25000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 20000, 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 9 | "batch_size": 2, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 2, 15 | "n_samples": 25, 16 | "early_stopping": -1, 17 | }, 18 | "model": { 19 | "name": "textattack/roberta-base-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/roberta/svs-5000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "end": 5000, 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 9 | "batch_size": 2, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 2, 15 | "n_samples": 25, 16 | "early_stopping": -1, 17 | }, 18 | "model": { 19 | "name": "textattack/roberta-base-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/xlnet/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerDeepLiftShap", 13 | }, 14 | "model": { 15 | "name": "textattack/xlnet-base-cased-imdb", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/imdb/xlnet/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientShap", 13 | "n_samples": 5, 14 | }, 15 | "model": { 16 | "name": "textattack/xlnet-base-cased-imdb", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/imdb/xlnet/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerGradientXActivation", 13 | }, 14 | "model": { 15 | "name": "textattack/xlnet-base-cased-imdb", 16 | "mode_load": "hf", 17 | "path_model": null, 18 | "tokenization": { 19 | "max_length": 512, 20 | "padding": "max_length", 21 | "return_tensors": "np", 22 | "truncation": true, 23 | "special_tokens_mask": true, 24 | } 25 | }, 26 | "visualization": { 27 | "columns": ["attributions", "predictions", "input_ids", "labels"], 28 | "gamma": 2.0, 29 | "normalize": true, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /configs/imdb/xlnet/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LayerIntegratedGradients", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | }, 16 | "model": { 17 | "name": "textattack/xlnet-base-cased-imdb", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/imdb/xlnet/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 100, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/xlnet-base-cased-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } -------------------------------------------------------------------------------- /configs/imdb/xlnet/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "LimeBase", 13 | "internal_batch_size": 1, 14 | "n_samples": 25, 15 | "mask_prob": 0.3, 16 | }, 17 | "model": { 18 | "name": "textattack/xlnet-base-cased-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } -------------------------------------------------------------------------------- /configs/imdb/xlnet/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 8 | "batch_size": 1, 9 | "root_dir": "$HOME/experiments/thermostat/datasets", 10 | }, 11 | "explainer": { 12 | "name": "Occlusion", 13 | "internal_batch_size": 1, 14 | "sliding_window_shapes": [3], 15 | }, 16 | "model": { 17 | "name": "textattack/xlnet-base-cased-imdb", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/imdb/xlnet/svs-10000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 5000, 8 | "end": 10000, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/xlnet-base-cased-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/xlnet/svs-15000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 10000, 8 | "end": 15000, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/xlnet-base-cased-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/xlnet/svs-20000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 15000, 8 | "end": 20000, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/xlnet-base-cased-imdb", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/imdb/xlnet/svs-25000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "start": 20000, 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/xlnet-base-cased-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/imdb/xlnet/svs-5000.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "imdb", 6 | "split": "test", 7 | "end": 5000, 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/xlnet-base-cased-imdb", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/mnli/albert/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerDeepLiftShap", 14 | }, 15 | "model": { 16 | "name": "prajjwal1/albert-base-v2-mnli", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "label"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/mnli/albert/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerGradientShap", 14 | "n_samples": 5, 15 | }, 16 | "model": { 17 | "name": "prajjwal1/albert-base-v2-mnli", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/mnli/albert/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerGradientXActivation", 14 | }, 15 | "model": { 16 | "name": "prajjwal1/albert-base-v2-mnli", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "label"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/mnli/albert/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerIntegratedGradients", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "prajjwal1/albert-base-v2-mnli", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "label"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/mnli/albert/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LimeBase", 14 | "internal_batch_size": 1, 15 | "n_samples": 100, 16 | "mask_prob": 0.3, 17 | }, 18 | "model": { 19 | "name": "prajjwal1/albert-base-v2-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/albert/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LimeBase", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | "mask_prob": 0.3, 17 | }, 18 | "model": { 19 | "name": "prajjwal1/albert-base-v2-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/albert/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 2, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "Occlusion", 14 | "internal_batch_size": 1, 15 | "sliding_window_shapes": [3], 16 | }, 17 | "model": { 18 | "name": "prajjwal1/albert-base-v2-mnli", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "label"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } -------------------------------------------------------------------------------- /configs/mnli/albert/svs-3250.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "end": 3250, 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "prajjwal1/albert-base-v2-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/albert/svs-9815.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "start": 6500, 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "prajjwal1/albert-base-v2-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/bert/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerDeepLiftShap", 14 | }, 15 | "model": { 16 | "name": "textattack/bert-base-uncased-MNLI", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "label"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/mnli/bert/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerGradientShap", 14 | "n_samples": 5, 15 | }, 16 | "model": { 17 | "name": "textattack/bert-base-uncased-MNLI", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/mnli/bert/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerGradientXActivation", 14 | }, 15 | "model": { 16 | "name": "textattack/bert-base-uncased-MNLI", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "label"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/mnli/bert/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerIntegratedGradients", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-MNLI", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "label"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/mnli/bert/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LimeBase", 14 | "internal_batch_size": 1, 15 | "n_samples": 100, 16 | "mask_prob": 0.3, 17 | }, 18 | "model": { 19 | "name": "textattack/bert-base-uncased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/bert/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LimeBase", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | "mask_prob": 0.3, 17 | }, 18 | "model": { 19 | "name": "textattack/bert-base-uncased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/bert/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "Occlusion", 14 | "internal_batch_size": 1, 15 | "sliding_window_shapes": [3], 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-MNLI", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/mnli/bert/svs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 2, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 2, 15 | "n_samples": 25, 16 | "early_stopping": -1, 17 | }, 18 | "model": { 19 | "name": "textattack/bert-base-uncased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/electra/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerDeepLiftShap", 14 | }, 15 | "model": { 16 | "name": "howey/electra-base-mnli", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/mnli/electra/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerGradientShap", 14 | "n_samples": 5, 15 | }, 16 | "model": { 17 | "name": "howey/electra-base-mnli", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/mnli/electra/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerGradientXActivation", 14 | }, 15 | "model": { 16 | "name": "howey/electra-base-mnli", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "label"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/mnli/electra/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerIntegratedGradients", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "howey/electra-base-mnli", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/mnli/electra/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LimeBase", 14 | "internal_batch_size": 1, 15 | "n_samples": 100, 16 | "mask_prob": 0.3, 17 | }, 18 | "model": { 19 | "name": "howey/electra-base-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/electra/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LimeBase", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | "mask_prob": 0.3, 17 | }, 18 | "model": { 19 | "name": "howey/electra-base-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/electra/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "Occlusion", 14 | "internal_batch_size": 1, 15 | "sliding_window_shapes": [3], 16 | }, 17 | "model": { 18 | "name": "howey/electra-base-mnli", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/mnli/electra/svs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "ShapleyValueSampling", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | "early_stopping": -1, 17 | }, 18 | "model": { 19 | "name": "howey/electra-base-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/roberta/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'special_tokens_mask', 'attention_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerDeepLiftShap", 14 | }, 15 | "model": { 16 | "name": "textattack/roberta-base-MNLI", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/mnli/roberta/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'special_tokens_mask', 'attention_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerGradientShap", 14 | "n_samples": 5, 15 | }, 16 | "model": { 17 | "name": "textattack/roberta-base-MNLI", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/mnli/roberta/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'special_tokens_mask', 'attention_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerGradientXActivation", 14 | }, 15 | "model": { 16 | "name": "textattack/roberta-base-MNLI", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/mnli/roberta/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'special_tokens_mask', 'attention_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerIntegratedGradients", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/roberta-base-MNLI", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/mnli/roberta/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LimeBase", 14 | "internal_batch_size": 1, 15 | "n_samples": 100, 16 | "mask_prob": 0.3, 17 | }, 18 | "model": { 19 | "name": "textattack/roberta-base-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } -------------------------------------------------------------------------------- /configs/mnli/roberta/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LimeBase", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | "mask_prob": 0.3, 17 | }, 18 | "model": { 19 | "name": "textattack/roberta-base-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } -------------------------------------------------------------------------------- /configs/mnli/roberta/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "Occlusion", 14 | "internal_batch_size": 1, 15 | "sliding_window_shapes": [3], 16 | }, 17 | "model": { 18 | "name": "textattack/roberta-base-MNLI", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/mnli/roberta/svs-3250.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "end": 3250, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/roberta-base-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/roberta/svs-9815.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "start": 6500, 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/roberta-base-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/xlnet/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerDeepLiftShap", 14 | }, 15 | "model": { 16 | "name": "textattack/xlnet-base-cased-MNLI", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/mnli/xlnet/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerGradientShap", 14 | "n_samples": 5, 15 | }, 16 | "model": { 17 | "name": "textattack/xlnet-base-cased-MNLI", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "labels"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/mnli/xlnet/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerGradientXActivation", 14 | }, 15 | "model": { 16 | "name": "textattack/xlnet-base-cased-MNLI", 17 | "mode_load": "hf", 18 | "path_model": null, 19 | "tokenization": { 20 | "max_length": 512, 21 | "padding": "max_length", 22 | "return_tensors": "np", 23 | "truncation": true, 24 | "special_tokens_mask": true, 25 | } 26 | }, 27 | "visualization": { 28 | "columns": ["attributions", "predictions", "input_ids", "labels"], 29 | "gamma": 2.0, 30 | "normalize": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /configs/mnli/xlnet/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LayerIntegratedGradients", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | }, 17 | "model": { 18 | "name": "textattack/xlnet-base-cased-MNLI", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/mnli/xlnet/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LimeBase", 14 | "internal_batch_size": 1, 15 | "n_samples": 100, 16 | "mask_prob": 0.3, 17 | }, 18 | "model": { 19 | "name": "textattack/xlnet-base-cased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/xlnet/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "LimeBase", 14 | "internal_batch_size": 1, 15 | "n_samples": 25, 16 | "mask_prob": 0.3, 17 | }, 18 | "model": { 19 | "name": "textattack/xlnet-base-cased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/xlnet/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 9 | "batch_size": 1, 10 | "root_dir": "$HOME/experiments/thermostat/datasets", 11 | }, 12 | "explainer": { 13 | "name": "Occlusion", 14 | "internal_batch_size": 1, 15 | "sliding_window_shapes": [3], 16 | }, 17 | "model": { 18 | "name": "textattack/xlnet-base-cased-MNLI", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "labels"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/mnli/xlnet/svs-3250.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "end": 3250, 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/xlnet-base-cased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/mnli/xlnet/svs-9815.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "multi_nli", 6 | "text_field": ["premise", "hypothesis"], 7 | "split": "validation_matched", 8 | "start": 6500, 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/xlnet-base-cased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "labels"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/qqp/albert/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "glue", 6 | "subset": "qqp", 7 | "split": "test", 8 | "end": 5000, 9 | "text_field": ["question1", "question2"], 10 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 11 | "batch_size": 1, 12 | "root_dir": "$HOME/experiments/thermostat/datasets", 13 | }, 14 | "explainer": { 15 | "name": "LayerGradientXActivation", 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-QQP", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "label"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } -------------------------------------------------------------------------------- /configs/qqp/bert/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "glue", 6 | "subset": "qqp", 7 | "split": "test", 8 | "end": 5000, 9 | "text_field": ["question1", "question2"], 10 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 11 | "batch_size": 1, 12 | "root_dir": "$HOME/experiments/thermostat/datasets", 13 | }, 14 | "explainer": { 15 | "name": "LayerGradientXActivation", 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-QQP", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "label"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } -------------------------------------------------------------------------------- /configs/xnli/albert/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerDeepLiftShap", 15 | }, 16 | "model": { 17 | "name": "prajjwal1/albert-base-v2-mnli", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/xnli/albert/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerGradientShap", 15 | "n_samples": 5, 16 | }, 17 | "model": { 18 | "name": "prajjwal1/albert-base-v2-mnli", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "label"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/xnli/albert/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerGradientXActivation", 15 | }, 16 | "model": { 17 | "name": "prajjwal1/albert-base-v2-mnli", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/xnli/albert/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerIntegratedGradients", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "prajjwal1/albert-base-v2-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/albert/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LimeBase", 15 | "internal_batch_size": 1, 16 | "n_samples": 100, 17 | "mask_prob": 0.3, 18 | }, 19 | "model": { 20 | "name": "prajjwal1/albert-base-v2-mnli", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "label"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/xnli/albert/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LimeBase", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | "mask_prob": 0.3, 18 | }, 19 | "model": { 20 | "name": "prajjwal1/albert-base-v2-mnli", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "label"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/xnli/albert/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "Occlusion", 15 | "internal_batch_size": 1, 16 | "sliding_window_shapes": [3], 17 | }, 18 | "model": { 19 | "name": "prajjwal1/albert-base-v2-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/albert/svs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "prajjwal1/albert-base-v2-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/bert/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerDeepLiftShap", 15 | }, 16 | "model": { 17 | "name": "textattack/bert-base-uncased-MNLI", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/xnli/bert/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerGradientShap", 15 | "n_samples": 5, 16 | }, 17 | "model": { 18 | "name": "textattack/bert-base-uncased-MNLI", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "label"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/xnli/bert/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerGradientXActivation", 15 | }, 16 | "model": { 17 | "name": "textattack/bert-base-uncased-MNLI", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } -------------------------------------------------------------------------------- /configs/xnli/bert/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerIntegratedGradients", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/bert-base-uncased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/bert/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LimeBase", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | "mask_prob": 0.3, 18 | }, 19 | "model": { 20 | "name": "textattack/bert-base-uncased-MNLI", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "label"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/xnli/bert/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "Occlusion", 15 | "internal_batch_size": 1, 16 | "sliding_window_shapes": [3], 17 | }, 18 | "model": { 19 | "name": "textattack/bert-base-uncased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/bert/svs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/bert-base-uncased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/electra/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerDeepLiftShap", 15 | }, 16 | "model": { 17 | "name": "howey/electra-base-mnli", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/xnli/electra/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerGradientShap", 15 | "n_samples": 5, 16 | }, 17 | "model": { 18 | "name": "howey/electra-base-mnli", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "label"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/xnli/electra/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerGradientXActivation", 15 | }, 16 | "model": { 17 | "name": "howey/electra-base-mnli", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/xnli/electra/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerIntegratedGradients", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "howey/electra-base-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/electra/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LimeBase", 15 | "internal_batch_size": 1, 16 | "n_samples": 100, 17 | "mask_prob": 0.3, 18 | }, 19 | "model": { 20 | "name": "howey/electra-base-mnli", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "label"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/xnli/electra/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LimeBase", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | "mask_prob": 0.3, 18 | }, 19 | "model": { 20 | "name": "howey/electra-base-mnli", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "label"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/xnli/electra/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "Occlusion", 15 | "internal_batch_size": 1, 16 | "sliding_window_shapes": [3], 17 | }, 18 | "model": { 19 | "name": "howey/electra-base-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/electra/svs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "howey/electra-base-mnli", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/roberta/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerDeepLiftShap", 15 | }, 16 | "model": { 17 | "name": "textattack/roberta-base-MNLI", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/xnli/roberta/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerGradientShap", 15 | "n_samples": 5, 16 | }, 17 | "model": { 18 | "name": "textattack/roberta-base-MNLI", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "label"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/xnli/roberta/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerGradientXActivation", 15 | }, 16 | "model": { 17 | "name": "textattack/roberta-base-MNLI", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/xnli/roberta/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerIntegratedGradients", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/roberta-base-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/roberta/lime-100.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LimeBase", 15 | "internal_batch_size": 1, 16 | "n_samples": 100, 17 | "mask_prob": 0.3, 18 | }, 19 | "model": { 20 | "name": "textattack/roberta-base-MNLI", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "label"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/xnli/roberta/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LimeBase", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | "mask_prob": 0.3, 18 | }, 19 | "model": { 20 | "name": "textattack/roberta-base-MNLI", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "label"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/xnli/roberta/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "Occlusion", 15 | "internal_batch_size": 1, 16 | "sliding_window_shapes": [3], 17 | }, 18 | "model": { 19 | "name": "textattack/roberta-base-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/roberta/svs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "text_field": ["premise", "hypothesis"], 8 | "split": "test", 9 | "columns": ['input_ids', 'attention_mask', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/roberta-base-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/xlnet/lds.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerDeepLiftShap", 15 | }, 16 | "model": { 17 | "name": "textattack/xlnet-base-cased-MNLI", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/xnli/xlnet/lgs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerGradientShap", 15 | "n_samples": 5, 16 | }, 17 | "model": { 18 | "name": "textattack/xlnet-base-cased-MNLI", 19 | "mode_load": "hf", 20 | "path_model": null, 21 | "tokenization": { 22 | "max_length": 512, 23 | "padding": "max_length", 24 | "return_tensors": "np", 25 | "truncation": true, 26 | "special_tokens_mask": true, 27 | } 28 | }, 29 | "visualization": { 30 | "columns": ["attributions", "predictions", "input_ids", "label"], 31 | "gamma": 2.0, 32 | "normalize": true, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /configs/xnli/xlnet/lgxa.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerGradientXActivation", 15 | }, 16 | "model": { 17 | "name": "textattack/xlnet-base-cased-MNLI", 18 | "mode_load": "hf", 19 | "path_model": null, 20 | "tokenization": { 21 | "max_length": 512, 22 | "padding": "max_length", 23 | "return_tensors": "np", 24 | "truncation": true, 25 | "special_tokens_mask": true, 26 | } 27 | }, 28 | "visualization": { 29 | "columns": ["attributions", "predictions", "input_ids", "label"], 30 | "gamma": 2.0, 31 | "normalize": true, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/xnli/xlnet/lig.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LayerIntegratedGradients", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/xlnet-base-cased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/xlnet/lime.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "LimeBase", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | "mask_prob": 0.3, 18 | }, 19 | "model": { 20 | "name": "textattack/xlnet-base-cased-MNLI", 21 | "mode_load": "hf", 22 | "path_model": null, 23 | "tokenization": { 24 | "max_length": 512, 25 | "padding": "max_length", 26 | "return_tensors": "np", 27 | "truncation": true, 28 | "special_tokens_mask": true, 29 | } 30 | }, 31 | "visualization": { 32 | "columns": ["attributions", "predictions", "input_ids", "label"], 33 | "gamma": 2.0, 34 | "normalize": true, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configs/xnli/xlnet/occ.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "Occlusion", 15 | "internal_batch_size": 1, 16 | "sliding_window_shapes": [3], 17 | }, 18 | "model": { 19 | "name": "textattack/xlnet-base-cased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /configs/xnli/xlnet/svs.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | "path": "$HOME/experiments/thermostat", 3 | "device": "cuda", 4 | "dataset": { 5 | "name": "xnli", 6 | "subset": "en", 7 | "split": "test", 8 | "text_field": ["premise", "hypothesis"], 9 | "columns": ['input_ids', 'attention_mask', 'token_type_ids', 'special_tokens_mask', 'labels'], 10 | "batch_size": 1, 11 | "root_dir": "$HOME/experiments/thermostat/datasets", 12 | }, 13 | "explainer": { 14 | "name": "ShapleyValueSampling", 15 | "internal_batch_size": 1, 16 | "n_samples": 25, 17 | }, 18 | "model": { 19 | "name": "textattack/xlnet-base-cased-MNLI", 20 | "mode_load": "hf", 21 | "path_model": null, 22 | "tokenization": { 23 | "max_length": 512, 24 | "padding": "max_length", 25 | "return_tensors": "np", 26 | "truncation": true, 27 | "special_tokens_mask": true, 28 | } 29 | }, 30 | "visualization": { 31 | "columns": ["attributions", "predictions", "input_ids", "label"], 32 | "gamma": 2.0, 33 | "normalize": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /download_data.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | 4 | from thermostat.data.readers import download_dataset 5 | from thermostat.utils import get_logger, read_config 6 | 7 | 8 | # Argument parser 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument('-c', help='Config file', 11 | default='configs/xnli/bert/lgxa.jsonnet') 12 | parser.add_argument('-home', help='Home directory', default=None) 13 | args = parser.parse_args() 14 | config_file = args.c 15 | home_dir = args.home 16 | 17 | logger = get_logger(name='download', file_out='./download.log', level=logging.INFO) 18 | 19 | config = read_config(config_file, home_dir) 20 | path_out = download_dataset(config['dataset'], logger) 21 | -------------------------------------------------------------------------------- /figures/imdb-429.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DFKI-NLP/thermostat/7abc285d038027290e3ee4d583713fcdff64012c/figures/imdb-429.png -------------------------------------------------------------------------------- /figures/instance-contents.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DFKI-NLP/thermostat/7abc285d038027290e3ee4d583713fcdff64012c/figures/instance-contents.png -------------------------------------------------------------------------------- /figures/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DFKI-NLP/thermostat/7abc285d038027290e3ee4d583713fcdff64012c/figures/logo.png -------------------------------------------------------------------------------- /figures/overview_v1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DFKI-NLP/thermostat/7abc285d038027290e3ee4d583713fcdff64012c/figures/overview_v1.png -------------------------------------------------------------------------------- /figures/spaces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DFKI-NLP/thermostat/7abc285d038027290e3ee4d583713fcdff64012c/figures/spaces.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | captum~=0.3.1 2 | datasets~=1.5.0 3 | jsonnet 4 | numpy~=1.22 5 | overrides~=3.1.0 6 | pandas 7 | protobuf 8 | pytorch-ignite 9 | scipy 10 | sentencepiece 11 | scikit-learn 12 | spacy~=3.0.6 13 | torch 14 | tqdm~=4.49.0 15 | transformers~=4.5.1 16 | -------------------------------------------------------------------------------- /run_explainer.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from thermostat.explain import explain_custom_data 3 | 4 | 5 | # Argument parser 6 | parser = argparse.ArgumentParser() 7 | parser.add_argument('-c', help='Config file', default='configs/imdb/bert/lig.jsonnet') 8 | parser.add_argument('-home', help='Home directory', default=None) 9 | args = parser.parse_args() 10 | config_file = args.c 11 | home_dir = args.home 12 | explain_custom_data(config_file, home_dir) 13 | -------------------------------------------------------------------------------- /src/thermostat/README.md: -------------------------------------------------------------------------------- 1 | # Details on the Thermostat package 2 | 3 | In this README, we describe our implementational details. 4 | More to follow. 5 | 6 | --- 7 | 8 | ### Config files 9 | jsonnet config files are used for generating the explanation datasets and have the following naming convention: 10 | `//.jsonnet` where 11 | * `` corresponds to a dataset (from `datasets` package by default, but can be any other locally stored dataset), 12 | * `` corresponds to an explainability method (usually provided through the `captum` package) and 13 | * `` corresponds to a model (from `transformers` package by default) -------------------------------------------------------------------------------- /src/thermostat/__init__.py: -------------------------------------------------------------------------------- 1 | from .data.dataset_utils import ( 2 | Thermopack, 3 | get_coordinate, 4 | load, 5 | ) 6 | from .data import thermostat_configs 7 | from .data import tokenization 8 | from .visualize import Heatmap 9 | 10 | __all__ = [Thermopack, get_coordinate, load, thermostat_configs, tokenization, Heatmap] 11 | -------------------------------------------------------------------------------- /src/thermostat/data/__init__.py: -------------------------------------------------------------------------------- 1 | from .readers import * 2 | -------------------------------------------------------------------------------- /src/thermostat/data/additional_configs.py: -------------------------------------------------------------------------------- 1 | CEN = ["contradiction", "entailment", "neutral"] 2 | CNE = ["contradiction", "neutral", "entailment"] 3 | ECN = ["entailment", "contradiction", "neutral"] 4 | ENC = ["entailment", "neutral", "contradiction"] # Legacy / Default from HF datasets 5 | NCE = ["neutral", "contradiction", "entailment"] 6 | NEC = ["neutral", "entailment", "contradiction"] 7 | 8 | LC_MAP = { 9 | "multi_nli-albert": CEN, 10 | "multi_nli-bert": CEN, 11 | "multi_nli-roberta": CNE, 12 | "multi_nli-xlnet": CEN, 13 | "xnli-albert": CEN, 14 | "xnli-bert": CEN, 15 | "xnli-roberta": CNE, 16 | "xnli-xlnet": CEN, 17 | } 18 | 19 | 20 | def get_label_names(config_name): 21 | cn_dataset_model = '-'.join(config_name.split('-')[:2]) 22 | if cn_dataset_model in LC_MAP.keys(): 23 | return LC_MAP[cn_dataset_model] 24 | -------------------------------------------------------------------------------- /src/thermostat/explainers/__init__.py: -------------------------------------------------------------------------------- 1 | from .grad import ( 2 | ExplainerLayerGradientXActivation, 3 | ExplainerLayerIntegratedGradients, 4 | ExplainerDeepLift, 5 | ) 6 | 7 | from .iba import ( 8 | ExplainerIBA, 9 | ) 10 | 11 | from .lime import ( 12 | ExplainerLime, 13 | ExplainerLimeBase, 14 | ) 15 | 16 | from .occlusion import ( 17 | ExplainerOcclusion 18 | ) 19 | 20 | from .svs import ( 21 | ExplainerShapleyValueSampling 22 | ) 23 | 24 | from .shap import ( 25 | ExplainerLayerDeepLiftShap, 26 | ExplainerLayerGradientShap, 27 | ) -------------------------------------------------------------------------------- /src/thermostat/streamlit/README.md: -------------------------------------------------------------------------------- 1 | # Exploring Thermostat with Streamlit 2 | 3 | This is based on an implementation by [Hugging Face](https://github.com/huggingface/datasets-viewer/tree/master) which they are using for their browser interface called [datasets viewer](https://huggingface.co/datasets/viewer/). 4 | 5 | 1. Setup a virtual environment and install our [PyPI package](https://pypi.org/project/thermostat-datasets/) `thermostat-datasets`. 6 | 2. Run `streamlit run run.py` from this directory which opens a new tab in your browser. 7 | 3. At the start, it will automatically download the `ag_news-albert-lgxa` configuration to your cache directory. This can take a while depending on your connection. 8 | 4. You can now freely explore all the Thermostat configurations using the sidebar dropdown menu. 9 | 10 | ![streamlit-menu](streamlit-menu.png) 11 | 12 | The checkbox "Show Attribution Scores" additionally gives you the option to add the feature attributions (rounded to three digits) as labels to each token. 13 | -------------------------------------------------------------------------------- /src/thermostat/streamlit/download.py: -------------------------------------------------------------------------------- 1 | import thermostat 2 | 3 | data = thermostat.load("imdb-bert-lig", cache_dir="~/datasets") 4 | -------------------------------------------------------------------------------- /src/thermostat/streamlit/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DFKI-NLP/thermostat/7abc285d038027290e3ee4d583713fcdff64012c/src/thermostat/streamlit/logo.png -------------------------------------------------------------------------------- /src/thermostat/streamlit/streamlit-menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DFKI-NLP/thermostat/7abc285d038027290e3ee4d583713fcdff64012c/src/thermostat/streamlit/streamlit-menu.png -------------------------------------------------------------------------------- /uc_expl-comp.py: -------------------------------------------------------------------------------- 1 | import thermostat 2 | 3 | unit_index = 378 4 | u_occ = thermostat.load("multi_nli-bert-occ")[unit_index] 5 | u_intg = thermostat.load("multi_nli-bert-lig")[unit_index] 6 | u_lime = thermostat.load("multi_nli-bert-lime")[unit_index] 7 | 8 | print(u_occ.predicted_label, u_occ.true_label) 9 | u_occ.render() 10 | print(u_intg.predicted_label, u_intg.true_label) 11 | u_intg.render() 12 | print(u_lime.predicted_label, u_lime.true_label) 13 | u_lime.render() 14 | -------------------------------------------------------------------------------- /uc_model-comp.py: -------------------------------------------------------------------------------- 1 | import thermostat 2 | 3 | bert = thermostat.load("multi_nli-bert-occ") 4 | electra = thermostat.load("multi_nli-electra-occ") 5 | 6 | for model_name, data in zip(["bert", "electra"], [bert, electra]): 7 | print(model_name) 8 | data.classification_report() 9 | print('=====================\n\n') 10 | 11 | disagreement = [(b, e) for (b, e) in zip(bert, electra) if b.predicted_label != e.predicted_label] 12 | 13 | # good examples in disagreement: 16 14 | contradictions = [(i, unit)[0] for i, unit in enumerate(disagreement) if unit[0].true_label == 'contradiction'] 15 | 16 | index = contradictions[16] 17 | u_b, u_e = disagreement[index] 18 | 19 | print(u_b.predicted_label, u_b.true_label) 20 | u_b.render() 21 | print(u_e.predicted_label, u_e.true_label) 22 | u_e.render() 23 | -------------------------------------------------------------------------------- /uc_rank-corr.py: -------------------------------------------------------------------------------- 1 | import thermostat 2 | from scipy.stats import kendalltau 3 | 4 | imdb_lime = thermostat.load("imdb-bert-lime") 5 | il_atts = imdb_lime.attributions.flatten() 6 | imdb_intg = thermostat.load("imdb-bert-lig") 7 | ii_atts = imdb_intg.attributions.flatten() 8 | kendall_imdb = kendalltau(il_atts, ii_atts) 9 | print(kendall_imdb) 10 | 11 | mnli_lime = thermostat.load("multi_nli-bert-lime") 12 | ml_atts = mnli_lime.attributions.flatten() 13 | mnli_intg = thermostat.load("multi_nli-bert-lig") 14 | mi_atts = mnli_intg.attributions.flatten() 15 | kendall_mnli = kendalltau(ml_atts, mi_atts) 16 | print(kendall_mnli) 17 | --------------------------------------------------------------------------------