├── .gitignore ├── LICENSE ├── README.md ├── bin ├── eval-template-with-descriptor.sh ├── eval-template.sh ├── few-shot-11b.sh ├── few-shot-3b.sh ├── few-shot-pretrained-11b-100k.sh ├── few-shot-pretrained-3b-100k.sh ├── fishmask-3b-part1.sh ├── fishmask-3b-part2.sh ├── full-model-3b.sh ├── pretrain_11b.sh ├── pretrain_3b.sh ├── raft_experiments.sh └── start.sh ├── configs ├── adapter.json ├── anli-r1.json ├── anli-r2.json ├── anli-r3.json ├── bitfit.json ├── cb.json ├── compacter.json ├── compacter_pp.json ├── copa.json ├── finetune.json ├── finetune_without_ln.json ├── finetune_without_ln_and_ul.json ├── finetune_without_ul.json ├── fishmask_002.json ├── fishmask_02.json ├── fishmask_train.json ├── h-swag.json ├── ia3.json ├── ia3_without_ul_and_ln.json ├── intrinsic_said_20k.json ├── intrinsic_said_500k.json ├── layernorm.json ├── lora.json ├── lora_scaling.json ├── lowrank_adapter.json ├── prefix_tuning.json ├── pretrain.json ├── prompt_tuning-100_prompts.json ├── prompt_tuning-10_prompts.json ├── prompt_tuning-unlikely_100_prompts.json ├── prompt_tuning-unlikely_10_prompts.json ├── rte.json ├── storycloze.json ├── t011b.json ├── t03b.json ├── wic.json ├── winogrande.json └── wsc.json ├── data └── few_shot │ ├── anli-r1 │ └── 50_shot │ │ ├── 0_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ ├── anli-r2 │ └── 50_shot │ │ ├── 0_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ ├── anli-r3 │ └── 50_shot │ │ ├── 0_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ ├── boolq │ └── 32_shot │ │ ├── 0_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ ├── cb │ └── 32_shot │ │ ├── 0_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ ├── copa │ └── 32_shot │ │ ├── 0_seed.jsonl │ │ ├── 100_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ ├── h-swag │ └── 20_shot │ │ ├── 0_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ ├── rte │ └── 32_shot │ │ ├── 0_seed.jsonl │ │ ├── 100_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ ├── sst2 │ └── 16_shot │ │ ├── 0_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ ├── storycloze │ └── 70_shot │ │ ├── 0_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ ├── wic │ └── 32_shot │ │ ├── 0_seed.jsonl │ │ ├── 100_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ ├── winogrande │ └── 50_shot │ │ ├── 0_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl │ └── wsc │ └── 32_shot │ ├── 0_seed.jsonl │ ├── 1024_seed.jsonl │ ├── 1_seed.jsonl │ ├── 32_seed.jsonl │ └── 42_seed.jsonl ├── pretrained_checkpoints ├── t011b_ia3_finish.pt └── t03b_ia3_finish.pt ├── requirements.txt └── src ├── __init__.py ├── data ├── __init__.py ├── data_module.py ├── dataset_readers.py ├── datasets.csv └── raft_prompt_construction_settings.jsonl ├── intrinsic_said_setup.py ├── models ├── AdapterVariants │ ├── Adapters.py │ └── VariantLayers.py ├── EncoderDecoder.py ├── adapters.py ├── bitfit.py ├── fishmask.py ├── fwh_cuda │ ├── fwh_cpp.cpp │ └── fwh_cu.cu ├── intrinsic.py ├── lora.py ├── modify_model.py ├── prefix_tuning.py └── prompt_tuning.py ├── pl_train.py ├── scripts ├── eval_template.py ├── few_glue.py └── get_result_table.py ├── ticl ├── icl_engines.py ├── model.py └── test_icl.py └── utils ├── Config.py ├── get_optimizer.py ├── get_scheduler.py └── util.py /.gitignore: -------------------------------------------------------------------------------- 1 | exp_out/ 2 | env/ 3 | results/ 4 | lib/ 5 | output/ 6 | pretrained_models 7 | pretrained_models/ 8 | .idea/ 9 | eche_ 10 | runs/ 11 | exp_out/ 12 | exp_probs/ 13 | *.pyc 14 | slurm* 15 | .installed.cfg 16 | develop-eggs 17 | dist 18 | downloads 19 | eggs 20 | parts 21 | src/*.egg-info 22 | lib 23 | lib64 24 | !src/data 25 | env.yaml 26 | untracked 27 | .so 28 | .o 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 r-three 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # T-Few 2 | 3 | This repository contains the official code for the paper: "[Few-Shot Parameter-Efficient Fine-Tuning is Better and Cheaper than In-Context Learning](https://arxiv.org/abs/2205.05638)". 4 | 5 | This method outperforms in-context learning with GPT-3 and achieves state-of-the-art on "[RAFT](https://huggingface.co/spaces/ought/raft-leaderboard)". 6 | 7 | ## Setup 8 | 9 | First, create a virtual environment for the project and install all the requirments. 10 | (We use conda to manage environments. Be sure to install and initialize conda first.) 11 | 12 | 1. Create a virtual environment with python 3.7 `conda create -n tfew python==3.7`, then activate the environment `conda activate tfew`. 13 | 2. Install other dependencies. `pip install -r requirements.txt -f https://download.pytorch.org/whl/cu113/torch_stable.html` 14 | 3. If you plan to run SAID, then install dependencies with `python src/intrinsic_said_setup.py develop`. Otherwise, skip this step. 15 | 16 | The steps above only needs to be done once. In addition, every time you start a new session, you will need to run `. bin/start.sh` 17 | 18 | ## Run your first experiment 19 | 20 | Once you finished setting up the environment, you can try running 21 | `CUDA_VISIBLE_DEVICES=3 python -m src.pl_train -c t0.json+rte.json -k save_model=False exp_name=first_exp` 22 | The outputs of this run will be saved to `${OUTPUT_PATH}/first_exp/`, which is usually `/t-few/exp_out/first_exp/`. Here, `first_exp` is the experiment name, you can run more experiments with different expeirment names. The code will automatically skip finished experiments. (However, if you wish to rerun a finished experiment under the same experiment name, you will need to manually remove the corresponding files in the output directory.) 23 | 24 | There are two ways to control an experiment. 25 | 26 | 1. You can specify config files with `-c`. Multiple config files can be combined with `+`. (When there are conflits, config terms from the config file on the right will have greater power.) This will be convinient when you have multiple terms that forms a fixed group. 27 | 2. You can override values with `-k`. This will be convinient when you need to change a small number of terms. 28 | 29 | It is recommended to use GPUs with 40GB to train T0(3B) and 80GB to train T0 30 | 31 | ## Run an array of experiments 32 | 33 | In this project, we often need to run a large number of experiments. 34 | Here is an example bash script `bin/few-shot-pretrained-3b-100k.sh` to fine-tune 3B pre-trained (IA)3 on all datasets. 35 | 36 | This should take a few hours. After that, you can use `scripts/get_results_table.py` to generate a csv summary. 37 | 38 | ## Citation 39 | 40 | If you find this repo helpful, welcome to cite our work: 41 | 42 | ``` 43 | @article{liu2020tfew, 44 | title={Few-Shot Parameter-Efficient Fine-Tuning is Better and Cheaper than In-Context Learning}, 45 | author={Liu, Haokun and Tam, Derek and Muqeeth, Mohammed and Mohta, Jay and Huang, Tenghao and Bansal, Mohit and Raffel, Colin}, 46 | journal={arXiv preprint arXiv:2205.05638}, 47 | year={2022} 48 | } 49 | ``` 50 | 51 | We use the following code in our works: 52 | 53 | ``` 54 | @article{mahabadi2021compacter, 55 | title={Compacter: Efficient low-rank hypercomplex adapter layers}, 56 | author={Mahabadi, Rabeeh Karimi and Henderson, James and Ruder, Sebastian}, 57 | journal={arXiv preprint arXiv:2106.04647}, 58 | year={2021} 59 | } 60 | 61 | @article{sung2021training, 62 | title={Training Neural Networks with Fixed Sparse Masks}, 63 | author={Sung, Yi-Lin and Nair, Varun and Raffel, Colin}, 64 | journal={arXiv preprint arXiv:2111.09839}, 65 | year={2021} 66 | } 67 | 68 | @article{aghajanyan2020intrinsic, 69 | title={Intrinsic dimensionality explains the effectiveness of language model fine-tuning}, 70 | author={Aghajanyan, Armen and Zettlemoyer, Luke and Gupta, Sonal}, 71 | journal={arXiv preprint arXiv:2012.13255}, 72 | year={2020} 73 | } 74 | ``` 75 | -------------------------------------------------------------------------------- /bin/eval-template-with-descriptor.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | seed=$1 6 | model=$2 7 | method=$3 8 | dataset=$4 9 | template_idx=$5 10 | descriptor=$6 11 | 12 | export orig_exp_name=${model}_${dataset}_seed${seed}_${method}_${descriptor} 13 | python -m src.pl_train -c ${model}.json+${method}.json+${dataset}.json -k load_weight=exp_out/${orig_exp_name}/finish.pt save_model=False exp_name=${orig_exp_name}_template_${template_idx} few_shot_random_seed=${seed} seed=${seed} compute_strategy="none" save_model=False allow_skip_exp=True num_steps=0 eval_template_idx=${template_idx} 14 | -------------------------------------------------------------------------------- /bin/eval-template.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | seed=$1 6 | model=$2 7 | method=$3 8 | dataset=$4 9 | template_idx=$5 10 | 11 | export orig_exp_name=${model}_${dataset}_seed${seed}_${method} 12 | python -m src.pl_train -c ${model}.json+${method}.json+${dataset}.json -k load_weight=exp_out/${orig_exp_name}/finish.pt save_model=False exp_name=${orig_exp_name}_template_${template_idx} few_shot_random_seed=${seed} seed=${seed} compute_strategy="none" save_model=False allow_skip_exp=True num_steps=0 eval_template_idx=${template_idx} 13 | -------------------------------------------------------------------------------- /bin/few-shot-11b.sh: -------------------------------------------------------------------------------- 1 | for seed in 1024 0 32 1 42 2 | do 3 | for dataset in copa h-swag storycloze winogrande wsc wic rte cb anli-r1 anli-r2 anli-r3 4 | do 5 | python -m src.pl_train -c t011b.json+ia3.json+${dataset}.json -k exp_name=t011b_${dataset}_seed${seed}_ia3 few_shot_random_seed=${seed} seed=${seed} eval_epoch_interval=50 batch_size=1 eval_batch_size=2 grad_accum_factor=8 allow_skip_exp=True 6 | done 7 | done 8 | 9 | -------------------------------------------------------------------------------- /bin/few-shot-3b.sh: -------------------------------------------------------------------------------- 1 | for method in ia3 lora lora_scaling bitfit layernorm compacter compacter_pp prompt_tuning-unlikely_100_prompts prompt_tuning-unlikely_10_prompts adapter intrinsic_said_20k intrinsic_said_500k prefix_tuning 2 | do 3 | for dataset in copa h-swag storycloze winogrande wsc wic rte cb anli-r1 anli-r2 anli-r3 4 | do 5 | for seed in 42 1024 0 1 32 6 | do 7 | python -m src.pl_train -c t03b.json+${dataset}.json+${method}.json -k exp_name=t03b_${dataset}_seed${seed}_${method} few_shot_random_seed=${seed} seed=${seed} 8 | done 9 | done 10 | done 11 | 12 | 13 | -------------------------------------------------------------------------------- /bin/few-shot-pretrained-11b-100k.sh: -------------------------------------------------------------------------------- 1 | for dataset in copa h-swag storycloze winogrande wsc wic rte cb anli-r1 anli-r2 anli-r3 2 | do 3 | for seed in 1024 0 32 1 42 4 | do 5 | python -m src.pl_train -c t011b.json+ia3.json+${dataset}.json -k load_weight="pretrained_checkpoints/t011b_ia3_finish.pt" exp_name=t011b_pretrained100k_${dataset}_seed${seed}_ia3 few_shot_random_seed=${seed} seed=${seed} eval_epoch_interval=50 batch_size=1 eval_batch_size=2 grad_accum_factor=8 6 | done 7 | done 8 | 9 | -------------------------------------------------------------------------------- /bin/few-shot-pretrained-3b-100k.sh: -------------------------------------------------------------------------------- 1 | for dataset in copa h-swag storycloze winogrande wsc wic rte cb anli-r1 anli-r2 anli-r3 2 | do 3 | for seed in 42 1024 0 1 32 4 | do 5 | python -m src.pl_train -c t03b.json+ia3.json+${dataset}.json -k load_weight="pretrained_checkpoints/t03b_ia3_finish.pt" exp_name=t03b_${dataset}_seed${seed}_ia3_pretrained100k few_shot_random_seed=${seed} seed=${seed} 6 | done 7 | done 8 | 9 | 10 | -------------------------------------------------------------------------------- /bin/fishmask-3b-part1.sh: -------------------------------------------------------------------------------- 1 | for method in fishmask_02 fishmask_002 2 | do 3 | for dataset in copa h-swag storycloze winogrande wsc wic rte cb anli-r1 anli-r2 anli-r3 4 | do 5 | for seed in 42 1024 0 1 32 6 | do 7 | python -m src.pl_train -c t03b.json+${dataset}.json+${method}.json -k exp_name=t03b_${dataset}_seed${seed}_${method} few_shot_random_seed=${seed} seed=${seed} 8 | done 9 | done 10 | done 11 | 12 | -------------------------------------------------------------------------------- /bin/fishmask-3b-part2.sh: -------------------------------------------------------------------------------- 1 | for method in fishmask_02 fishmask_002 2 | do 3 | for dataset in copa h-swag storycloze winogrande wsc wic rte cb anli-r1 anli-r2 anli-r3 4 | do 5 | for seed in 42 1024 0 1 32 6 | do 7 | python -m src.pl_train -c t03b.json+${dataset}.json+fishmask_train.json -k exp_name=t03b_${dataset}_seed${seed}_${method}_train few_shot_random_seed=${seed} seed=${seed} eval_epoch_interval=50 batch_size=4 grad_accum_factor=2 fishmask_path=exp_out/t03b_${dataset}_seed${seed}_${method}/mask.bin 8 | done 9 | done 10 | done 11 | 12 | -------------------------------------------------------------------------------- /bin/full-model-3b.sh: -------------------------------------------------------------------------------- 1 | for method in finetune finetune_without_ln finetune_without_ul finetune_without_ln_and_ul 2 | do 3 | for dataset in copa h-swag storycloze winogrande wsc wic rte cb anli-r1 anli-r2 anli-r3 4 | do 5 | for seed in 42 1024 0 1 32 6 | do 7 | python -m src.pl_train -c t03b.json+${dataset}.json+${method}.json -k exp_name=t03b_${dataset}_seed${seed}_${method} few_shot_random_seed=${seed} seed=${seed} eval_epoch_interval=50 8 | done 9 | done 10 | done 11 | 12 | 13 | -------------------------------------------------------------------------------- /bin/pretrain_11b.sh: -------------------------------------------------------------------------------- 1 | CUDA_VISIBLE_DEVICES=0,1,2,3 python -m src.pl_train -c t011b.json+ia3.json+pretrain.json -k compute_strategy="ddp" exp_name=t011b_pretrain_ia3 batch_size=4 2 | -------------------------------------------------------------------------------- /bin/pretrain_3b.sh: -------------------------------------------------------------------------------- 1 | CUDA_VISIBLE_DEVICES=0,1 python -m src.pl_train -c t03b.json+ia3.json+pretrain.json -k compute_strategy="ddp" exp_name=t03b_pretrain_ia3 allow_skip_exp=False batch_size=8 -------------------------------------------------------------------------------- /bin/raft_experiments.sh: -------------------------------------------------------------------------------- 1 | for dataset in ade_corpus_v2 terms_of_service tai_safety_research neurips_impact_statement_risks overruling systematic_review_inclusion one_stop_english tweet_eval_hate twitter_complaints semiconductor_org_typesdo 2 | python -m src.pl_train -c t011b.json+ia3.json -k load_weight=\"/root/nicl/exp_out/finish.pt\" dataset=${dataset} exp_name=t011b_${dataset}_ia3 few_shot=False unlikely_loss=1 length_norm=1 mc_loss=1 batch_size=1 grad_accum_factor=8 compute_strategy="none" raft_cross_validation=False 3 | python -m src.pl_train -c t011b.json+ia3.json -k load_weight=\"/root/nicl/exp_out/t011b_${dataset}_ia3/finish.pt\" dataset=${dataset} exp_name=t011b_${dataset}_ia3_eval few_shot=False length_norm=1 raft_cross_validation=False compute_strategy="none" eval_batch_size=1 num_steps=0 save_model=False 4 | done 5 | 6 | 7 | python -m src.pl_train -c t011b.json+ia3.json -k load_weight=\"/root/nicl/exp_out/finish.pt\" dataset=banking_77 exp_name=t011b_banking_77_ia3 few_shot=False unlikely_loss=0 length_norm=1 mc_loss=0 max_seq_len=512 batch_size=1 grad_accum_factor=8 compute_strategy="none" raft_cross_validation=False 8 | python -m src.pl_train -c t011b.json+ia3.json -k load_weight=\"/root/nicl/exp_out/t011b_banking_77_ia3/finish.pt\" dataset=banking_77 exp_name=t011b_banking_77_ia3_eval few_shot=False length_norm=1 eval_batch_size=1 raft_cross_validation=True compute_strategy="none" num_steps=0 raft_labels_in_input_string=comma cleaned_answer_choices_b77=1 save_model=False max_seq_len=512 -------------------------------------------------------------------------------- /bin/start.sh: -------------------------------------------------------------------------------- 1 | conda activate tfew 2 | export NICL_ROOT=`pwd` 3 | export PYTHONPATH=$NICL_ROOT:$PYTHONPATH 4 | export OUTPUT_PATH=$NICL_ROOT/exp_out 5 | export CONFIG_PATH=$NICL_ROOT/configs -------------------------------------------------------------------------------- /configs/adapter.json: -------------------------------------------------------------------------------- 1 | { 2 | "adapter_type": "normal", 3 | "adapter_reduction_factor": 32, 4 | "adapter_non_linearity": "relu", 5 | "normal_adapter_residual": true, 6 | "trainable_param_names": ".*layer_norm.*|.*adapter.*", 7 | "model_modifier": "adapters", 8 | "lr": 3e-3, 9 | "num_steps": 500 10 | } -------------------------------------------------------------------------------- /configs/anli-r1.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "anli-r1", 3 | "batch_size": 8, 4 | "eval_batch_size": 16, 5 | "grad_accum_factor": 1, 6 | "num_shot": 50, 7 | "length_norm": 1, 8 | "mc_loss": 1, 9 | "unlikely_loss": 1 10 | } -------------------------------------------------------------------------------- /configs/anli-r2.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "anli-r2", 3 | "batch_size": 8, 4 | "eval_batch_size": 16, 5 | "grad_accum_factor": 1, 6 | "num_shot": 50, 7 | "length_norm": 1, 8 | "mc_loss": 1, 9 | "unlikely_loss": 1 10 | } -------------------------------------------------------------------------------- /configs/anli-r3.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "anli-r3", 3 | "batch_size": 8, 4 | "eval_batch_size": 16, 5 | "grad_accum_factor": 1, 6 | "num_shot": 50, 7 | "length_norm": 1, 8 | "mc_loss": 1, 9 | "unlikely_loss": 1 10 | } -------------------------------------------------------------------------------- /configs/bitfit.json: -------------------------------------------------------------------------------- 1 | { 2 | "bitfit_modules": ".*", 3 | "bitfit_layers": "q|k|v|o|wi_[01]|w_o", 4 | "trainable_param_names" : ".*bias|.*layer_norm.*", 5 | "model_modifier": "bitfit", 6 | "num_steps": 1000, 7 | "lr": 3e-4 8 | } 9 | -------------------------------------------------------------------------------- /configs/cb.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "cb", 3 | "batch_size": 8, 4 | "eval_batch_size": 16, 5 | "grad_accum_factor": 1, 6 | "num_shot": 32, 7 | "length_norm": 1, 8 | "mc_loss": 1, 9 | "unlikely_loss": 1 10 | } -------------------------------------------------------------------------------- /configs/compacter.json: -------------------------------------------------------------------------------- 1 | { 2 | "adapter_type": "compacter", 3 | "adapter_reduction_factor": 32, 4 | "adapter_non_linearity": "relu", 5 | "compacter_hypercomplex_division": 4, 6 | "compacter_learn_phm": true, 7 | "compacter_hypercomplex_nonlinearity": "glorot-uniform", 8 | "compacter_shared_phm_rule": false, 9 | "compacter_factorized_phm": true, 10 | "compacter_shared_W_phm": false, 11 | "compacter_factorized_phm_rule": false, 12 | "compacter_phm_c_init": "normal", 13 | "compacter_phm_rank": 1, 14 | "compacter_phm_init_range": 0.0001, 15 | "compacter_add_compacter_in_self_attention": true, 16 | "compacter_add_compacter_in_cross_attention": false, 17 | "compacter_kronecker_prod": false, 18 | "trainable_param_names": ".*layer_norm.*|.*adapter.*", 19 | "model_modifier": "adapters", 20 | "lr": 3e-3, 21 | "num_steps": 500 22 | } 23 | -------------------------------------------------------------------------------- /configs/compacter_pp.json: -------------------------------------------------------------------------------- 1 | { 2 | "adapter_type": "compacter", 3 | "adapter_reduction_factor": 32, 4 | "adapter_non_linearity": "relu", 5 | "compacter_hypercomplex_division": 4, 6 | "compacter_learn_phm": true, 7 | "compacter_hypercomplex_nonlinearity": "glorot-uniform", 8 | "compacter_shared_phm_rule": false, 9 | "compacter_factorized_phm": true, 10 | "compacter_shared_W_phm": false, 11 | "compacter_factorized_phm_rule": false, 12 | "compacter_phm_c_init": "normal", 13 | "compacter_phm_rank": 1, 14 | "compacter_phm_init_range": 0.0001, 15 | "compacter_kronecker_prod": false, 16 | "compacter_add_compacter_in_self_attention": false, 17 | "compacter_add_compacter_in_cross_attention": false, 18 | "trainable_param_names": ".*layer_norm.*|.*adapter.*", 19 | "model_modifier": "adapters", 20 | "lr": 3e-3, 21 | "num_steps": 500 22 | } 23 | -------------------------------------------------------------------------------- /configs/copa.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "copa", 3 | "batch_size": 8, 4 | "eval_batch_size": 16, 5 | "grad_accum_factor": 1, 6 | "num_shot": 32, 7 | "length_norm": 1, 8 | "mc_loss": 1, 9 | "unlikely_loss": 1 10 | } -------------------------------------------------------------------------------- /configs/finetune.json: -------------------------------------------------------------------------------- 1 | { 2 | "num_steps": 300, 3 | "lr": 3e-4 4 | } 5 | -------------------------------------------------------------------------------- /configs/finetune_without_ln.json: -------------------------------------------------------------------------------- 1 | { 2 | "num_steps": 300, 3 | "lr": 3e-4, 4 | "length_norm": 0, 5 | "mc_loss": 0 6 | } 7 | -------------------------------------------------------------------------------- /configs/finetune_without_ln_and_ul.json: -------------------------------------------------------------------------------- 1 | { 2 | "num_steps": 300, 3 | "lr": 3e-4, 4 | "length_norm": 0, 5 | "mc_loss": 0, 6 | "unlikely_loss": 0 7 | } 8 | -------------------------------------------------------------------------------- /configs/finetune_without_ul.json: -------------------------------------------------------------------------------- 1 | { 2 | "num_steps": 300, 3 | "lr": 3e-4, 4 | "unlikely_loss": 0 5 | } 6 | -------------------------------------------------------------------------------- /configs/fishmask_002.json: -------------------------------------------------------------------------------- 1 | { 2 | "fishmask_mode": "create", 3 | "fishmask_keep_ratio": 0.0002, 4 | "batch_size": 1, 5 | "eval_before_training": false, 6 | "eval_epoch_interval": 10000, 7 | "save_model": false 8 | } -------------------------------------------------------------------------------- /configs/fishmask_02.json: -------------------------------------------------------------------------------- 1 | { 2 | "fishmask_mode": "create", 3 | "fishmask_keep_ratio": 0.002, 4 | "batch_size": 1, 5 | "eval_before_training": false, 6 | "eval_epoch_interval": 10000, 7 | "save_model": false 8 | } -------------------------------------------------------------------------------- /configs/fishmask_train.json: -------------------------------------------------------------------------------- 1 | { 2 | "fishmask_path": null, 3 | "fishmask_mode": "apply", 4 | "num_steps": 1500 5 | } -------------------------------------------------------------------------------- /configs/h-swag.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "h-swag", 3 | "batch_size": 4, 4 | "eval_batch_size": 16, 5 | "grad_accum_factor": 2, 6 | "num_shot": 20, 7 | "length_norm": 1, 8 | "mc_loss": 1, 9 | "unlikely_loss": 1 10 | } -------------------------------------------------------------------------------- /configs/ia3.json: -------------------------------------------------------------------------------- 1 | { 2 | "lora_scaling_rank": 1, 3 | "lora_rank": 0, 4 | "lora_init_scale": 0.0, 5 | "lora_modules": ".*SelfAttention|.*EncDecAttention|.*DenseReluDense", 6 | "lora_layers": "k|v|wi_1.*", 7 | "trainable_param_names": ".*lora_b.*", 8 | "model_modifier": "lora", 9 | "lr": 3e-3, 10 | "num_steps": 1000 11 | } -------------------------------------------------------------------------------- /configs/ia3_without_ul_and_ln.json: -------------------------------------------------------------------------------- 1 | { 2 | "lora_scaling_rank": 1, 3 | "lora_rank": 0, 4 | "lora_init_scale": 0.0, 5 | "lora_modules": ".*SelfAttention|.*EncDecAttention|.*DenseReluDense", 6 | "lora_layers": "k|v|wi_1.*", 7 | "trainable_param_names": ".*lora_b.*", 8 | "model_modifier": "lora", 9 | "lr": 3e-3, 10 | "length_norm": 0, 11 | "mc_loss": 0, 12 | "unlikely_loss": 0, 13 | "num_steps": 1000 14 | } -------------------------------------------------------------------------------- /configs/intrinsic_said_20k.json: -------------------------------------------------------------------------------- 1 | { 2 | "intrinsic_projection": "fastfood", 3 | "intrinsic_said": true, 4 | "intrinsic_dim": 20000, 5 | "trainable_param_names":"intrinsic_parameter|intrinsic_parameter_said", 6 | "model_modifier": "intrinsic", 7 | "lr": 3e-2, 8 | "num_steps": 3000 9 | } -------------------------------------------------------------------------------- /configs/intrinsic_said_500k.json: -------------------------------------------------------------------------------- 1 | { 2 | "intrinsic_projection": "fastfood", 3 | "intrinsic_said": true, 4 | "intrinsic_dim": 500000, 5 | "trainable_param_names":"intrinsic_parameter|intrinsic_parameter_said", 6 | "model_modifier": "intrinsic", 7 | "lr": 3e-2, 8 | "num_steps": 3000 9 | } -------------------------------------------------------------------------------- /configs/layernorm.json: -------------------------------------------------------------------------------- 1 | { 2 | "trainable_param_names": ".*layer_norm.*", 3 | "lr": 3e-4, 4 | "num_steps": 1000 5 | } -------------------------------------------------------------------------------- /configs/lora.json: -------------------------------------------------------------------------------- 1 | { 2 | "lora_rank": 4, 3 | "lora_init_scale": 0.01, 4 | "lora_modules": ".*SelfAttention|.*EncDecAttention|.*DenseReluDense", 5 | "lora_layers": "q|k|v|o|w.*", 6 | "trainable_param_names": ".*layer_norm.*|.*lora_[ab].*", 7 | "model_modifier": "lora", 8 | "lr": 3e-3, 9 | "num_steps": 1000 10 | } -------------------------------------------------------------------------------- /configs/lora_scaling.json: -------------------------------------------------------------------------------- 1 | { 2 | "lora_scaling_rank": 1, 3 | "lora_rank": 0, 4 | "lora_init_scale": 0.0, 5 | "lora_modules": ".*SelfAttention|.*EncDecAttention|.*DenseReluDense", 6 | "lora_layers": "q|k|v|o|w.*", 7 | "trainable_param_names": ".*lora_[ab].*", 8 | "model_modifier": "lora", 9 | "lr": 3e-3, 10 | "num_steps": 1000 11 | } -------------------------------------------------------------------------------- /configs/lowrank_adapter.json: -------------------------------------------------------------------------------- 1 | { 2 | "adapter_type": "lowrank", 3 | "adapter_reduction_factor": 4, 4 | "adapter_non_linearity": "relu", 5 | "lowrank_adapter_w_init": "glorot-uniform", 6 | "lowrank_adapter_rank": 1, 7 | "trainable_param_names": ".*layer_norm.*|.*adapter.*", 8 | "model_modifier": "adapters", 9 | "lr": 3e-3, 10 | "num_steps": 500 11 | } 12 | -------------------------------------------------------------------------------- /configs/prefix_tuning.json: -------------------------------------------------------------------------------- 1 | { 2 | "prefix_tuning_init_path": "exp_out/prefix_tuning_init.pt", 3 | "prefix_tuning_init_text": "Question:\nAnswer:", 4 | "prefix_tuning_parameterization": "mlp-512", 5 | "trainable_param_names": ".*prefix_.*", 6 | "model_modifier": "prefix-tuning", 7 | "lr": 1e-2, 8 | "num_steps": 1000 9 | } -------------------------------------------------------------------------------- /configs/pretrain.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "T0Mixture", 3 | "batch_size": 2, 4 | "grad_accum_factor": 1, 5 | "few_shot": false, 6 | "eval_before_training": false, 7 | "eval_epoch_interval": 10000, 8 | "num_steps": 100000, 9 | "lr": 3e-4 10 | } -------------------------------------------------------------------------------- /configs/prompt_tuning-100_prompts.json: -------------------------------------------------------------------------------- 1 | { 2 | "prompt_tuning_num_prefix_emb": 100, 3 | "model_modifier": "prompt-tuning", 4 | "lr": 3e-1, 5 | "unlikely_loss": 0, 6 | "batch_size": 8, 7 | "num_steps": 1000, 8 | "trainable_param_names": ".*prompt_embedding.*", 9 | "scale_parameter": false 10 | } 11 | -------------------------------------------------------------------------------- /configs/prompt_tuning-10_prompts.json: -------------------------------------------------------------------------------- 1 | { 2 | "prompt_tuning_num_prefix_emb": 10, 3 | "model_modifier": "prompt-tuning", 4 | "lr": 3e-1, 5 | "unlikely_loss": 0, 6 | "batch_size": 8, 7 | "num_steps": 1000, 8 | "trainable_param_names": ".*prompt_embedding.*", 9 | "scale_parameter": false 10 | } 11 | -------------------------------------------------------------------------------- /configs/prompt_tuning-unlikely_100_prompts.json: -------------------------------------------------------------------------------- 1 | { 2 | "prompt_tuning_num_prefix_emb": 100, 3 | "model_modifier": "prompt-tuning", 4 | "lr": 3e-1, 5 | "batch_size": 4, 6 | "grad_accum_factor": 2, 7 | "num_steps": 1000, 8 | "trainable_param_names": ".*prompt_embedding.*", 9 | "scale_parameter": false 10 | } 11 | -------------------------------------------------------------------------------- /configs/prompt_tuning-unlikely_10_prompts.json: -------------------------------------------------------------------------------- 1 | { 2 | "prompt_tuning_num_prefix_emb": 10, 3 | "model_modifier": "prompt-tuning", 4 | "lr": 3e-1, 5 | "batch_size": 8, 6 | "num_steps": 1000, 7 | "trainable_param_names": ".*prompt_embedding.*", 8 | "scale_parameter": false 9 | } 10 | -------------------------------------------------------------------------------- /configs/rte.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "rte", 3 | "batch_size": 8, 4 | "eval_batch_size": 16, 5 | "grad_accum_factor": 1, 6 | "num_shot": 32, 7 | "length_norm": 1, 8 | "mc_loss": 1, 9 | "unlikely_loss": 1 10 | } -------------------------------------------------------------------------------- /configs/storycloze.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "storycloze", 3 | "batch_size": 8, 4 | "eval_batch_size": 16, 5 | "grad_accum_factor": 1, 6 | "num_shot": 70, 7 | "length_norm": 1, 8 | "mc_loss": 1, 9 | "unlikely_loss": 1 10 | } -------------------------------------------------------------------------------- /configs/t011b.json: -------------------------------------------------------------------------------- 1 | { 2 | "exp_name": "debug", 3 | "origin_model": "bigscience/T0", 4 | "compute_precision": "bf16", 5 | "compute_strategy": "none", 6 | "allow_skip_exp":false 7 | } 8 | -------------------------------------------------------------------------------- /configs/t03b.json: -------------------------------------------------------------------------------- 1 | { 2 | "exp_name": "debug", 3 | "origin_model": "bigscience/T0_3B", 4 | "compute_precision": "bf16", 5 | "compute_strategy": "none" 6 | } -------------------------------------------------------------------------------- /configs/wic.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "wic", 3 | "batch_size": 8, 4 | "eval_batch_size": 16, 5 | "grad_accum_factor": 1, 6 | "num_shot": 32, 7 | "length_norm": 1, 8 | "mc_loss": 1, 9 | "unlikely_loss": 1 10 | } -------------------------------------------------------------------------------- /configs/winogrande.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "winogrande", 3 | "batch_size": 8, 4 | "eval_batch_size": 16, 5 | "grad_accum_factor": 1, 6 | "num_shot": 50, 7 | "length_norm": 1, 8 | "mc_loss": 1, 9 | "unlikely_loss": 1 10 | } -------------------------------------------------------------------------------- /configs/wsc.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": "wsc", 3 | "batch_size": 8, 4 | "eval_batch_size": 16, 5 | "grad_accum_factor": 1, 6 | "num_shot": 32, 7 | "length_norm": 1, 8 | "mc_loss": 1, 9 | "unlikely_loss": 1 10 | } -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/0_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"premise": "The group overlooked the woman's faux pas.", "choice1": "The woman was relieved.", "choice2": "The woman apologized.", "question": "effect", "idx": 132, "label": 0} 2 | {"premise": "The woman complimented the man.", "choice1": "He sneezed.", "choice2": "He blushed.", "question": "effect", "idx": 309, "label": 1} 3 | {"premise": "The girl shared her lunch with her friend.", "choice1": "Her friend forgot to bring his lunch.", "choice2": "Her friend sat next to her at lunch.", "question": "cause", "idx": 341, "label": 0} 4 | {"premise": "My flight got delayed.", "choice1": "Tornadoes were sighted in the area.", "choice2": "I was held up at airport security.", "question": "cause", "idx": 196, "label": 0} 5 | {"premise": "The woman pressed down on the piano key.", "choice1": "A sound played.", "choice2": "The key got stuck.", "question": "effect", "idx": 246, "label": 0} 6 | {"premise": "The man threw his empty can onto the street.", "choice1": "He was jumped from behind.", "choice2": "He was fined for littering.", "question": "effect", "idx": 60, "label": 1} 7 | {"premise": "I chugged the pitcher of water.", "choice1": "I walked upstairs.", "choice2": "I ran five miles.", "question": "cause", "idx": 155, "label": 1} 8 | {"premise": "The man revealed personal information to the therapist.", "choice1": "He trusted the therapist.", "choice2": "He disagreed with the therapist.", "question": "cause", "idx": 261, "label": 0} 9 | {"premise": "The mirror in the bathroom fogged up.", "choice1": "The girl turned on the fan.", "choice2": "The girl applied her makeup.", "question": "effect", "idx": 141, "label": 0} 10 | {"premise": "The fortune teller's revelations were convincing.", "choice1": "She received many returning customers.", "choice2": "Many customers asked for refunds.", "question": "effect", "idx": 214, "label": 0} 11 | {"premise": "The woman won the lottery.", "choice1": "She bought a yacht.", "choice2": "She joined a church.", "question": "effect", "idx": 37, "label": 0} 12 | {"premise": "The man became disabled.", "choice1": "His family offered him financial support.", "choice2": "His family cut off contact with him.", "question": "effect", "idx": 134, "label": 0} 13 | {"premise": "The surfer caught the wave.", "choice1": "The wave carried her to the shore.", "choice2": "She paddled her board into the ocean.", "question": "effect", "idx": 113, "label": 0} 14 | {"premise": "The hunter ran out of ammunition.", "choice1": "He reloaded the gun.", "choice2": "He aimed at the deer.", "question": "effect", "idx": 348, "label": 0} 15 | {"premise": "The pond froze over for the winter.", "choice1": "People skated on the pond.", "choice2": "People brought boats to the pond.", "question": "effect", "idx": 12, "label": 0} 16 | {"premise": "The crowd gave the band a standing ovation.", "choice1": "The band signed autographs.", "choice2": "The band reappeared on the stage.", "question": "effect", "idx": 59, "label": 1} 17 | {"premise": "The egg splattered.", "choice1": "I dropped it.", "choice2": "I boiled it.", "question": "cause", "idx": 293, "label": 0} 18 | {"premise": "I held the door for the woman entering behind me.", "choice1": "The door slammed shut.", "choice2": "The woman thanked me.", "question": "effect", "idx": 140, "label": 1} 19 | {"premise": "The tenants were evicted from the apartment.", "choice1": "They neglected to pay their rent.", "choice2": "They got along with their landlord.", "question": "cause", "idx": 206, "label": 0} 20 | {"premise": "The woman felt an electric shock.", "choice1": "She grabbed the doorknob.", "choice2": "She pounded on the door.", "question": "cause", "idx": 199, "label": 0} 21 | {"premise": "The man held his breath.", "choice1": "He went underwater.", "choice2": "He treaded water.", "question": "cause", "idx": 176, "label": 0} 22 | {"premise": "The chef pressed down on the dough.", "choice1": "The dough crumbled.", "choice2": "The dough flattened.", "question": "effect", "idx": 268, "label": 1} 23 | {"premise": "The turtle hid in its shell.", "choice1": "It ate a fish.", "choice2": "It detected a predator.", "question": "cause", "idx": 124, "label": 1} 24 | {"premise": "The cup of tea was scalding hot.", "choice1": "I blew on it.", "choice2": "I poured it out.", "question": "effect", "idx": 344, "label": 0} 25 | {"premise": "The father shut off the children's television.", "choice1": "It was bedtime for the children.", "choice2": "The children were watching cartoons.", "question": "cause", "idx": 175, "label": 0} 26 | {"premise": "The woman fanned herself with her hand.", "choice1": "She installed the air conditioner in the room.", "choice2": "The air conditioner in the room broke.", "question": "cause", "idx": 313, "label": 1} 27 | {"premise": "The girl was angry with her friend.", "choice1": "The girl spread a rumor about her friend.", "choice2": "The girl told a secret to her friend.", "question": "effect", "idx": 78, "label": 0} 28 | {"premise": "The girl gasped.", "choice1": "Her friend stuck an ice cube down her back.", "choice2": "Her friend gave her a pat on the back.", "question": "cause", "idx": 15, "label": 0} 29 | {"premise": "The boy had trouble buttoning his shirt.", "choice1": "He refused to wear the shirt.", "choice2": "He asked his mother for help.", "question": "effect", "idx": 286, "label": 1} 30 | {"premise": "The child kicked the stack of blocks.", "choice1": "The stack towered over the boy's head.", "choice2": "The blocks scattered all over the rug.", "question": "effect", "idx": 102, "label": 1} 31 | {"premise": "The motorcyclist twisted the throttle.", "choice1": "The motorcycle coasted to a halt.", "choice2": "The motorcycle shot forward.", "question": "effect", "idx": 170, "label": 1} 32 | {"premise": "The ocean tide was dangerous.", "choice1": "The swimmers put on more sunscreen.", "choice2": "The swimmers returned towards the shore.", "question": "effect", "idx": 303, "label": 1} 33 | -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/100_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"premise": "The vehicles travelled slowly.", "choice1": "It was cloudy.", "choice2": "It was foggy.", "question": "cause", "idx": 248, "label": 1} 2 | {"premise": "The administrator cleared her throat.", "choice1": "The meeting was postponed.", "choice2": "The meeting commenced.", "question": "effect", "idx": 318, "label": 1} 3 | {"premise": "The boy climbed the tree.", "choice1": "A cat got stuck in the tree.", "choice2": "A bird built a nest in the tree.", "question": "cause", "idx": 173, "label": 0} 4 | {"premise": "My daughter suffered an allergy attack.", "choice1": "I encouraged her to take deep breaths.", "choice2": "I took her to the emergency room.", "question": "effect", "idx": 285, "label": 1} 5 | {"premise": "The woman felt ashamed of scar on her face.", "choice1": "She explained the scar to strangers.", "choice2": "She hid the scar with makeup.", "question": "effect", "idx": 184, "label": 1} 6 | {"premise": "I wanted to lighten the mood of the conversation.", "choice1": "I remained quiet.", "choice2": "I told a joke.", "question": "effect", "idx": 235, "label": 1} 7 | {"premise": "The turtle hid in its shell.", "choice1": "It ate a fish.", "choice2": "It detected a predator.", "question": "cause", "idx": 124, "label": 1} 8 | {"premise": "The boy was afraid to go to bed.", "choice1": "He turned on his nightlight.", "choice2": "He watched a horror movie.", "question": "cause", "idx": 282, "label": 1} 9 | {"premise": "We found empty seats in the theatre.", "choice1": "The film began.", "choice2": "We sat down.", "question": "effect", "idx": 151, "label": 1} 10 | {"premise": "The dust came off the desk.", "choice1": "I wiped the desk with a cloth.", "choice2": "I removed the books from the desk.", "question": "cause", "idx": 169, "label": 0} 11 | {"premise": "The girl handed down her clothes to her younger sister.", "choice1": "The clothes were tattered.", "choice2": "She outgrew the clothes.", "question": "cause", "idx": 125, "label": 1} 12 | {"premise": "The man took a shower.", "choice1": "He was sweaty from working out.", "choice2": "He opened a new bar of soap.", "question": "cause", "idx": 296, "label": 0} 13 | {"premise": "The police closed the investigation.", "choice1": "They apprehended the suspect.", "choice2": "The victim recovered.", "question": "cause", "idx": 342, "label": 0} 14 | {"premise": "The teenager ran away from home.", "choice1": "He relied on his parents.", "choice2": "His parents abused him.", "question": "cause", "idx": 103, "label": 1} 15 | {"premise": "The city re-elected the mayor.", "choice1": "The city cut its budget.", "choice2": "The mayor fulfilled his promises.", "question": "cause", "idx": 370, "label": 1} 16 | {"premise": "The man made an error in his calculations.", "choice1": "He checked his work.", "choice2": "His work was hasty.", "question": "cause", "idx": 278, "label": 1} 17 | {"premise": "The man broke his toe.", "choice1": "He got a hole in his sock.", "choice2": "He dropped a hammer on his foot.", "question": "cause", "idx": 236, "label": 1} 18 | {"premise": "The girl stared at herself in the mirror.", "choice1": "She felt self-conscious.", "choice2": "The mirror was smudged.", "question": "cause", "idx": 315, "label": 0} 19 | {"premise": "The woman awakened.", "choice1": "She took sleeping pills.", "choice2": "Her husband was snoring.", "question": "cause", "idx": 96, "label": 1} 20 | {"premise": "The man's eye became infected.", "choice1": "He went blind.", "choice2": "He put on glasses.", "question": "effect", "idx": 43, "label": 0} 21 | {"premise": "The doctors amputated the patient's leg.", "choice1": "It was badly bruised.", "choice2": "It was severely infected.", "question": "cause", "idx": 314, "label": 1} 22 | {"premise": "The woman sensed a pleasant smell.", "choice1": "She remembered to take her medication.", "choice2": "She was reminded of her childhood.", "question": "effect", "idx": 97, "label": 1} 23 | {"premise": "The girl was angry with her friend.", "choice1": "The girl spread a rumor about her friend.", "choice2": "The girl told a secret to her friend.", "question": "effect", "idx": 78, "label": 0} 24 | {"premise": "I held the door for the woman entering behind me.", "choice1": "The door slammed shut.", "choice2": "The woman thanked me.", "question": "effect", "idx": 140, "label": 1} 25 | {"premise": "The man took notice of the woman.", "choice1": "He caught a whiff of her perfume.", "choice2": "He had his back turned to her.", "question": "cause", "idx": 272, "label": 0} 26 | {"premise": "The boy turned the lid on the jar.", "choice1": "The jar became warm.", "choice2": "The lid came off.", "question": "effect", "idx": 153, "label": 1} 27 | {"premise": "The parents left their children with a babysitter.", "choice1": "Their youngest child started attending preschool.", "choice2": "They made plans to celebrate their anniversary.", "question": "cause", "idx": 127, "label": 1} 28 | {"premise": "The man wanted to get revenge on his enemy.", "choice1": "The man framed his enemy for a crime.", "choice2": "The man declared a truce with his enemy.", "question": "effect", "idx": 275, "label": 0} 29 | {"premise": "The woman felt compelled to help someone in need.", "choice1": "She donated blood.", "choice2": "She wrote a poem.", "question": "effect", "idx": 56, "label": 0} 30 | {"premise": "The host cancelled the party.", "choice1": "She was certain she had the flu.", "choice2": "She worried she would catch the flu.", "question": "cause", "idx": 23, "label": 0} 31 | {"premise": "The bar closed.", "choice1": "It was crowded.", "choice2": "It was 3 AM.", "question": "cause", "idx": 98, "label": 1} 32 | {"premise": "The family postponed the barbeque.", "choice1": "The forecast called for storms.", "choice2": "It was a holiday weekend.", "question": "cause", "idx": 242, "label": 0} 33 | -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"premise": "The speaker disarmed her debate opponent.", "choice1": "The speaker congratulated her opponent.", "choice2": "The speaker won the debate.", "question": "effect", "idx": 114, "label": 1} 2 | {"premise": "The girl wanted to demonstrate proper etiquette.", "choice1": "She threw away her napkin after eating.", "choice2": "She put her napkin on her lap before eating.", "question": "effect", "idx": 218, "label": 1} 3 | {"premise": "The woman hit a midlife crisis.", "choice1": "She retired.", "choice2": "She went sky diving.", "question": "effect", "idx": 372, "label": 1} 4 | {"premise": "The pharmaceutical company recalled the drug.", "choice1": "Users reported dangerous side effects.", "choice2": "Users got refills on their prescriptions.", "question": "cause", "idx": 105, "label": 0} 5 | {"premise": "The glasses on the table spilled.", "choice1": "The child drew on the table.", "choice2": "The child yanked the tablecloth.", "question": "cause", "idx": 368, "label": 1} 6 | {"premise": "The woman repaired her faucet.", "choice1": "The faucet was leaky.", "choice2": "The faucet was turned off.", "question": "cause", "idx": 10, "label": 0} 7 | {"premise": "The cup of tea was scalding hot.", "choice1": "I blew on it.", "choice2": "I poured it out.", "question": "effect", "idx": 344, "label": 0} 8 | {"premise": "The girl desired her parent's approval.", "choice1": "She ran away from home.", "choice2": "She obeyed her parent's rules.", "question": "effect", "idx": 388, "label": 1} 9 | {"premise": "The photograph faded.", "choice1": "It was fake.", "choice2": "It was old.", "question": "cause", "idx": 345, "label": 1} 10 | {"premise": "The water in the teapot started to boil.", "choice1": "The teapot cooled.", "choice2": "The teapot whistled.", "question": "effect", "idx": 369, "label": 1} 11 | {"premise": "The doctor x-rayed the patient's arm.", "choice1": "He discovered that the patient's arm was broken.", "choice2": "He put the patient's arm in a cast.", "question": "effect", "idx": 185, "label": 0} 12 | {"premise": "Air pollution in the city worsened.", "choice1": "Factories increased their production.", "choice2": "Factories shut down.", "question": "cause", "idx": 392, "label": 0} 13 | {"premise": "The rain subsided.", "choice1": "I went for a walk.", "choice2": "I browsed the internet.", "question": "effect", "idx": 343, "label": 0} 14 | {"premise": "The girl pushed her friend.", "choice1": "Her friend choked.", "choice2": "Her friend fell over.", "question": "effect", "idx": 294, "label": 1} 15 | {"premise": "The toddler became cranky.", "choice1": "Her mother put her down for a nap.", "choice2": "Her mother fixed her hair into pigtails.", "question": "effect", "idx": 26, "label": 0} 16 | {"premise": "My body cast a shadow over the grass.", "choice1": "The sun was rising.", "choice2": "The grass was cut.", "question": "cause", "idx": 0, "label": 0} 17 | {"premise": "The retail shop shut down.", "choice1": "Its sales were lagging.", "choice2": "It renewed its lease.", "question": "cause", "idx": 390, "label": 0} 18 | {"premise": "The vehicles travelled slowly.", "choice1": "It was cloudy.", "choice2": "It was foggy.", "question": "cause", "idx": 248, "label": 1} 19 | {"premise": "The man caught his shirtsleeve on the rungs of the fence.", "choice1": "He hopped the fence.", "choice2": "The sleeve ripped.", "question": "effect", "idx": 361, "label": 1} 20 | {"premise": "The photo albums collected dust in the attic.", "choice1": "They remained untouched for years.", "choice2": "The family reminisced over the photos.", "question": "cause", "idx": 239, "label": 0} 21 | {"premise": "The girl stared at herself in the mirror.", "choice1": "She felt self-conscious.", "choice2": "The mirror was smudged.", "question": "cause", "idx": 315, "label": 0} 22 | {"premise": "The authorities vowed to protect the identity of the crime victim.", "choice1": "The victim struggled to recall details about the crime.", "choice2": "They withheld the victim's name from the public.", "question": "effect", "idx": 72, "label": 1} 23 | {"premise": "The woman was summoned for jury duty.", "choice1": "She contacted her lawyer.", "choice2": "She cancelled her appointments.", "question": "effect", "idx": 312, "label": 1} 24 | {"premise": "The waistband on my pants was loose.", "choice1": "I put on a belt.", "choice2": "I put on a hat.", "question": "effect", "idx": 204, "label": 0} 25 | {"premise": "The football coach lost his voice after the game.", "choice1": "He yelled whenever a player fumbled the ball.", "choice2": "He whistled whenever a player scored a touchdown.", "question": "cause", "idx": 308, "label": 0} 26 | {"premise": "The group overlooked the woman's faux pas.", "choice1": "The woman was relieved.", "choice2": "The woman apologized.", "question": "effect", "idx": 132, "label": 0} 27 | {"premise": "The shop was closed.", "choice1": "The owner was helping customers.", "choice2": "The shop was undergoing renovation.", "question": "cause", "idx": 68, "label": 1} 28 | {"premise": "The boy climbed the tree.", "choice1": "A cat got stuck in the tree.", "choice2": "A bird built a nest in the tree.", "question": "cause", "idx": 173, "label": 0} 29 | {"premise": "My mailbox was overflowing with letters.", "choice1": "The lid on the mailbox broke.", "choice2": "I went on vacation for two weeks.", "question": "cause", "idx": 266, "label": 1} 30 | {"premise": "The screen on the laptop went black.", "choice1": "The laptop's warranty expired.", "choice2": "The laptop's battery died.", "question": "cause", "idx": 297, "label": 1} 31 | {"premise": "Several witnesses of the crime testified against the suspect.", "choice1": "The suspect was acquitted.", "choice2": "The suspect was convicted.", "question": "effect", "idx": 126, "label": 1} 32 | {"premise": "The elderly woman suffered a stroke.", "choice1": "The woman's daughter came over to clean her house.", "choice2": "The woman's daughter moved in to take care of her.", "question": "effect", "idx": 11, "label": 1} 33 | -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/1_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"premise": "The man's watch was broken.", "choice1": "He waited at the bus stop.", "choice2": "He asked a stranger for the time.", "question": "effect", "idx": 398, "label": 1} 2 | {"premise": "The girl handed down her clothes to her younger sister.", "choice1": "The clothes were tattered.", "choice2": "She outgrew the clothes.", "question": "cause", "idx": 125, "label": 1} 3 | {"premise": "The DJ turned the music on.", "choice1": "People left the party.", "choice2": "People began to dance.", "question": "effect", "idx": 328, "label": 1} 4 | {"premise": "The fussy baby calmed down.", "choice1": "The parents shook a rattle in front of the baby.", "choice2": "The parents picked out a name for the baby.", "question": "cause", "idx": 339, "label": 0} 5 | {"premise": "The host served dinner to his guests.", "choice1": "His guests were gracious.", "choice2": "His guests went hungry.", "question": "effect", "idx": 172, "label": 0} 6 | {"premise": "The police closed the investigation.", "choice1": "They apprehended the suspect.", "choice2": "The victim recovered.", "question": "cause", "idx": 342, "label": 0} 7 | {"premise": "The smoke alarm went off.", "choice1": "I lit a candle.", "choice2": "I burnt my dinner.", "question": "cause", "idx": 197, "label": 1} 8 | {"premise": "I scrubbed the stain on the floor.", "choice1": "I got rid of the stain.", "choice2": "I got dirt on the floor.", "question": "effect", "idx": 291, "label": 0} 9 | {"premise": "The friends' debate dragged on interminably.", "choice1": "The friends saw eye to eye.", "choice2": "The friends were splitting hairs.", "question": "cause", "idx": 29, "label": 1} 10 | {"premise": "The boy found the television show boring.", "choice1": "He recorded it.", "choice2": "He shut it off.", "question": "effect", "idx": 284, "label": 1} 11 | {"premise": "The chair squeaked.", "choice1": "The man carried the chair to his office.", "choice2": "The man scooted the chair closer to his desk.", "question": "cause", "idx": 174, "label": 1} 12 | {"premise": "The woman hit a midlife crisis.", "choice1": "She retired.", "choice2": "She went sky diving.", "question": "effect", "idx": 372, "label": 1} 13 | {"premise": "The girl performed in a dance recital.", "choice1": "Her parents showed her how to dance.", "choice2": "Her parents came to watch the recital.", "question": "effect", "idx": 188, "label": 1} 14 | {"premise": "The boy giggled uncontrollably.", "choice1": "His brother tickled him.", "choice2": "His brother kicked him.", "question": "cause", "idx": 324, "label": 0} 15 | {"premise": "The customer thought the souvenir was overpriced.", "choice1": "The vender sold it to him.", "choice2": "He bargained with the vendor.", "question": "effect", "idx": 321, "label": 1} 16 | {"premise": "The car looked filthy.", "choice1": "The owner took it to the car wash.", "choice2": "He owner took it in for a paint job.", "question": "effect", "idx": 227, "label": 0} 17 | {"premise": "The woman exaggerated the details of the story.", "choice1": "She wanted attention.", "choice2": "She felt shy.", "question": "cause", "idx": 371, "label": 0} 18 | {"premise": "The politician lost the election.", "choice1": "He ran negative campaign ads.", "choice2": "No one voted for him.", "question": "cause", "idx": 5, "label": 1} 19 | {"premise": "The girl was angry with her friend.", "choice1": "The girl spread a rumor about her friend.", "choice2": "The girl told a secret to her friend.", "question": "effect", "idx": 78, "label": 0} 20 | {"premise": "The man uncovered incriminating evidence against his enemy.", "choice1": "The man avoided his enemy.", "choice2": "The man blackmailed his enemy.", "question": "effect", "idx": 223, "label": 1} 21 | {"premise": "The inside of the train went dark.", "choice1": "The train travelled through a tunnel.", "choice2": "The train ran off the tracks.", "question": "cause", "idx": 122, "label": 0} 22 | {"premise": "The family postponed the barbeque.", "choice1": "The forecast called for storms.", "choice2": "It was a holiday weekend.", "question": "cause", "idx": 242, "label": 0} 23 | {"premise": "The man wanted to save money.", "choice1": "He withdrew money from his savings account.", "choice2": "He cut back on making frivolous purchases.", "question": "effect", "idx": 382, "label": 1} 24 | {"premise": "The fortune teller's revelations were convincing.", "choice1": "She received many returning customers.", "choice2": "Many customers asked for refunds.", "question": "effect", "idx": 214, "label": 0} 25 | {"premise": "It got dark outside.", "choice1": "Snowflakes began to fall from the sky.", "choice2": "The moon became visible in the sky.", "question": "effect", "idx": 17, "label": 1} 26 | {"premise": "The window was opaque.", "choice1": "The glass was stained.", "choice2": "The blinds were open.", "question": "cause", "idx": 92, "label": 0} 27 | {"premise": "The father put his daughter in her stroller.", "choice1": "She learned to walk.", "choice2": "He was tired of carrying her.", "question": "cause", "idx": 366, "label": 1} 28 | {"premise": "I jumped instinctively in fright.", "choice1": "I sensed someone approaching me from behind.", "choice2": "I heard people chuckling in the adjacent room.", "question": "cause", "idx": 201, "label": 0} 29 | {"premise": "The man caught his shirtsleeve on the rungs of the fence.", "choice1": "He hopped the fence.", "choice2": "The sleeve ripped.", "question": "effect", "idx": 361, "label": 1} 30 | {"premise": "The man woke up with a toothache.", "choice1": "He missed an appointment with the dentist.", "choice2": "He made an appointment with the dentist.", "question": "effect", "idx": 207, "label": 1} 31 | {"premise": "The woman spoke with a foreign accent.", "choice1": "She came from a wealthy family.", "choice2": "She migrated from another country.", "question": "cause", "idx": 81, "label": 1} 32 | {"premise": "The guests of the party hid behind the couch.", "choice1": "It was a surprise party.", "choice2": "It was a birthday party.", "question": "cause", "idx": 4, "label": 0} 33 | -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/32_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"premise": "The man uncovered incriminating evidence against his enemy.", "choice1": "The man avoided his enemy.", "choice2": "The man blackmailed his enemy.", "question": "effect", "idx": 223, "label": 1} 2 | {"premise": "The navy bombed the ship.", "choice1": "The ship crashed into the pier.", "choice2": "The ship's debris sunk in the sea.", "question": "effect", "idx": 145, "label": 1} 3 | {"premise": "I ironed my shirt.", "choice1": "It was clean.", "choice2": "It was wrinkled.", "question": "cause", "idx": 146, "label": 1} 4 | {"premise": "The father put his daughter in her stroller.", "choice1": "She learned to walk.", "choice2": "He was tired of carrying her.", "question": "cause", "idx": 366, "label": 1} 5 | {"premise": "The chef pressed down on the dough.", "choice1": "The dough crumbled.", "choice2": "The dough flattened.", "question": "effect", "idx": 268, "label": 1} 6 | {"premise": "I was confused by the professor's lecture.", "choice1": "I paid attention to the professor.", "choice2": "I asked the professor questions.", "question": "effect", "idx": 109, "label": 1} 7 | {"premise": "The army surrendered the battle.", "choice1": "They were outnumbered.", "choice2": "They stormed the city.", "question": "cause", "idx": 144, "label": 0} 8 | {"premise": "The cup of tea was scalding hot.", "choice1": "I blew on it.", "choice2": "I poured it out.", "question": "effect", "idx": 344, "label": 0} 9 | {"premise": "The librarian located the book in the stacks.", "choice1": "I requested the librarian's help.", "choice2": "The book was checked out.", "question": "cause", "idx": 384, "label": 0} 10 | {"premise": "The driver pulled over to the side of the road.", "choice1": "He went through an intersection.", "choice2": "He noticed a stranded vehicle.", "question": "cause", "idx": 148, "label": 1} 11 | {"premise": "The parents forbade their children from watching the movie. ", "choice1": "The movie was rated R.", "choice2": "The movie was animated.", "question": "cause", "idx": 209, "label": 0} 12 | {"premise": "The hospital sent the patient home.", "choice1": "The patient's symptoms cleared up.", "choice2": "The patient's family visited him.", "question": "cause", "idx": 149, "label": 0} 13 | {"premise": "The woman wrote a check to the gas company.", "choice1": "She preferred paying by cash.", "choice2": "She received her monthly bill.", "question": "cause", "idx": 300, "label": 1} 14 | {"premise": "The boy felt homesick.", "choice1": "He went away to camp.", "choice2": "He bickered with his sister.", "question": "cause", "idx": 193, "label": 0} 15 | {"premise": "The woman felt compelled to help someone in need.", "choice1": "She donated blood.", "choice2": "She wrote a poem.", "question": "effect", "idx": 56, "label": 0} 16 | {"premise": "The boy's hands were shaking.", "choice1": "He planned to ask the girl out.", "choice2": "His palms were cold and sweaty.", "question": "cause", "idx": 118, "label": 0} 17 | {"premise": "The man signed the document.", "choice1": "The transaction was voided.", "choice2": "The transaction became official.", "question": "effect", "idx": 54, "label": 1} 18 | {"premise": "The man revealed personal information to the therapist.", "choice1": "He trusted the therapist.", "choice2": "He disagreed with the therapist.", "question": "cause", "idx": 261, "label": 0} 19 | {"premise": "The fugitive hid from the police.", "choice1": "The police dropped the case.", "choice2": "The fugitive remained at large.", "question": "effect", "idx": 79, "label": 1} 20 | {"premise": "Air leaked out of the beach ball.", "choice1": "It was inflated.", "choice2": "There was a hole in it.", "question": "cause", "idx": 243, "label": 1} 21 | {"premise": "The woman wanted to be a doctor.", "choice1": "She visited the hospital.", "choice2": "She went to medical school.", "question": "effect", "idx": 283, "label": 1} 22 | {"premise": "The offender violated parole.", "choice1": "She was sent back to jail.", "choice2": "She stole money from a church.", "question": "effect", "idx": 13, "label": 0} 23 | {"premise": "The woman fanned herself with her hand.", "choice1": "She installed the air conditioner in the room.", "choice2": "The air conditioner in the room broke.", "question": "cause", "idx": 313, "label": 1} 24 | {"premise": "The family postponed the barbeque.", "choice1": "The forecast called for storms.", "choice2": "It was a holiday weekend.", "question": "cause", "idx": 242, "label": 0} 25 | {"premise": "The gardener pulled the weeds.", "choice1": "The weeds came out of the soil.", "choice2": "The weeds produced seeds.", "question": "effect", "idx": 357, "label": 0} 26 | {"premise": "The fortune teller's revelations were convincing.", "choice1": "She received many returning customers.", "choice2": "Many customers asked for refunds.", "question": "effect", "idx": 214, "label": 0} 27 | {"premise": "The woman awakened.", "choice1": "She took sleeping pills.", "choice2": "Her husband was snoring.", "question": "cause", "idx": 96, "label": 1} 28 | {"premise": "The boy murmured his reply to his mother.", "choice1": "His mother told him to whisper.", "choice2": "His mother told him to speak up.", "question": "effect", "idx": 195, "label": 1} 29 | {"premise": "The school principal implemented a dress code.", "choice1": "Students protested the decision.", "choice2": "Students were expelled from school.", "question": "effect", "idx": 233, "label": 0} 30 | {"premise": "The woman's ring slipped off in the shower.", "choice1": "The woman polished the ring.", "choice2": "The ring went down the drain.", "question": "effect", "idx": 19, "label": 1} 31 | {"premise": "The climbers reached the peak of the mountain.", "choice1": "They encountered an avalanche.", "choice2": "They congratulated each other.", "question": "effect", "idx": 212, "label": 1} 32 | {"premise": "The girl politely declined the hamburger.", "choice1": "She was a vegetarian.", "choice2": "She liked fast food.", "question": "cause", "idx": 290, "label": 0} 33 | -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/42_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"premise": "The parents forbade their children from watching the movie. ", "choice1": "The movie was rated R.", "choice2": "The movie was animated.", "question": "cause", "idx": 209, "label": 0} 2 | {"premise": "The shopper wondered about the cost of the item.", "choice1": "He checked its price tag.", "choice2": "He decided to buy it.", "question": "effect", "idx": 280, "label": 0} 3 | {"premise": "The patient was dehydrated.", "choice1": "The nurse tested his reflexes.", "choice2": "The nurse gave him an IV.", "question": "effect", "idx": 33, "label": 1} 4 | {"premise": "The woman deleted the email.", "choice1": "The sender tried to solicit money from her.", "choice2": "The sender requested a response from her.", "question": "cause", "idx": 210, "label": 0} 5 | {"premise": "I slipped on the floor.", "choice1": "The tile was wet.", "choice2": "The tile was cracked.", "question": "cause", "idx": 93, "label": 0} 6 | {"premise": "The service at the restaurant was slow.", "choice1": "There were many empty tables.", "choice2": "The restaurant was crowded.", "question": "cause", "idx": 84, "label": 1} 7 | {"premise": "The security guard replayed the surveillance footage.", "choice1": "The surveillance camera was out of focus.", "choice2": "He noticed some suspicious activity.", "question": "cause", "idx": 329, "label": 1} 8 | {"premise": "The man slid the razor across his chin.", "choice1": "His stubble grew.", "choice2": "His stubble disappeared.", "question": "effect", "idx": 94, "label": 1} 9 | {"premise": "My mailbox was overflowing with letters.", "choice1": "The lid on the mailbox broke.", "choice2": "I went on vacation for two weeks.", "question": "cause", "idx": 266, "label": 1} 10 | {"premise": "Several witnesses of the crime testified against the suspect.", "choice1": "The suspect was acquitted.", "choice2": "The suspect was convicted.", "question": "effect", "idx": 126, "label": 1} 11 | {"premise": "The customer filed a complaint with the store manager.", "choice1": "The sales associate undercharged the customer.", "choice2": "The sales associate acted rude to the customer.", "question": "cause", "idx": 9, "label": 1} 12 | {"premise": "The man caught his shirtsleeve on the rungs of the fence.", "choice1": "He hopped the fence.", "choice2": "The sleeve ripped.", "question": "effect", "idx": 361, "label": 1} 13 | {"premise": "The woman felt compelled to help someone in need.", "choice1": "She donated blood.", "choice2": "She wrote a poem.", "question": "effect", "idx": 56, "label": 0} 14 | {"premise": "The authorities vowed to protect the identity of the crime victim.", "choice1": "The victim struggled to recall details about the crime.", "choice2": "They withheld the victim's name from the public.", "question": "effect", "idx": 72, "label": 1} 15 | {"premise": "The group overlooked the woman's faux pas.", "choice1": "The woman was relieved.", "choice2": "The woman apologized.", "question": "effect", "idx": 132, "label": 0} 16 | {"premise": "The executive decided not to hire the applicant.", "choice1": "The applicant failed a background check.", "choice2": "The applicant had experience for the job.", "question": "cause", "idx": 42, "label": 0} 17 | {"premise": "The man made an error in his calculations.", "choice1": "He checked his work.", "choice2": "His work was hasty.", "question": "cause", "idx": 278, "label": 1} 18 | {"premise": "I made a resolution to eat a healthy diet.", "choice1": "I attended a yoga class.", "choice2": "I bought fruits and vegetables.", "question": "effect", "idx": 376, "label": 1} 19 | {"premise": "The woman visited her family.", "choice1": "She distrusted them.", "choice2": "She missed them.", "question": "cause", "idx": 231, "label": 1} 20 | {"premise": "The mother became eye level with her son.", "choice1": "She squatted down.", "choice2": "She held his hand.", "question": "cause", "idx": 385, "label": 0} 21 | {"premise": "The man dropped food on the floor.", "choice1": "His dog jumped up on him.", "choice2": "His dog ran over to eat the food.", "question": "effect", "idx": 77, "label": 1} 22 | {"premise": "The girl gasped.", "choice1": "Her friend stuck an ice cube down her back.", "choice2": "Her friend gave her a pat on the back.", "question": "cause", "idx": 15, "label": 0} 23 | {"premise": "The swimmer won the race.", "choice1": "She received a gold medal.", "choice2": "Her team lost the meet.", "question": "effect", "idx": 391, "label": 0} 24 | {"premise": "The woman betrayed her friend.", "choice1": "Her friend sent her a greeting card.", "choice2": "Her friend cut off contact with her.", "question": "effect", "idx": 271, "label": 1} 25 | {"premise": "My body cast a shadow over the grass.", "choice1": "The sun was rising.", "choice2": "The grass was cut.", "question": "cause", "idx": 0, "label": 0} 26 | {"premise": "The print on the brochure was tiny.", "choice1": "The man put his glasses on.", "choice2": "The man retrieved a pen from his pocket.", "question": "effect", "idx": 396, "label": 0} 27 | {"premise": "The speaker disarmed her debate opponent.", "choice1": "The speaker congratulated her opponent.", "choice2": "The speaker won the debate.", "question": "effect", "idx": 114, "label": 1} 28 | {"premise": "My jaw dropped open.", "choice1": "I was unhappy.", "choice2": "I was shocked.", "question": "cause", "idx": 225, "label": 1} 29 | {"premise": "The man fell unconscious.", "choice1": "The assailant struck the man in the head.", "choice2": "The assailant took the man's wallet.", "question": "cause", "idx": 262, "label": 0} 30 | {"premise": "I botched the job interview.", "choice1": "The interviewer asked difficult questions.", "choice2": "I anticipated the interviewer's questions.", "question": "cause", "idx": 104, "label": 0} 31 | {"premise": "The skier slipped on the slope.", "choice1": "She dropped her ski poles.", "choice2": "She hit a patch of ice.", "question": "cause", "idx": 395, "label": 1} 32 | {"premise": "The boy felt homesick.", "choice1": "He went away to camp.", "choice2": "He bickered with his sister.", "question": "cause", "idx": 193, "label": 0} 33 | -------------------------------------------------------------------------------- /data/few_shot/sst2/16_shot/0_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"sentence": "deny its seriousness and quality ", "label": 0, "idx": 63456} 2 | {"sentence": "completely lacking in chills ", "label": 0, "idx": 11437} 3 | {"sentence": "this sade is hardly a perverse , dangerous libertine and agitator -- which would have made for better drama . ", "label": 0, "idx": 49380} 4 | {"sentence": "were dylan thomas alive to witness first-time director ethan hawke 's strained chelsea walls , he might have been tempted to change his landmark poem to , ` do not go gentle into that good theatre . ' ", "label": 0, "idx": 26110} 5 | {"sentence": "defined by childlike dimness and a handful of quirks ", "label": 0, "idx": 62356} 6 | {"sentence": "neo-nazism ", "label": 0, "idx": 17974} 7 | {"sentence": "this odd , distant portuguese import more or less borrows from bad lieutenant and les vampires , and comes up with a kind of art-house gay porn film . ", "label": 0, "idx": 53211} 8 | {"sentence": "when twentysomething hotsies make movies about their lives , hard-driving narcissism is a given ", "label": 0, "idx": 28131} 9 | {"sentence": "though it 's not very well shot or composed or edited ", "label": 0, "idx": 59001} 10 | {"sentence": "paper-thin supporting characters ", "label": 0, "idx": 13403} 11 | {"sentence": "schaeffer 's film never settles into the light-footed enchantment the material needs , and the characters ' quirks and foibles never jell into charm ", "label": 0, "idx": 63296} 12 | {"sentence": "perversity ", "label": 0, "idx": 56183} 13 | {"sentence": "'m not sure which will take longer to heal : the welt on johnny knoxville 's stomach from a riot-control projectile or my own tortured psyche . ", "label": 0, "idx": 3738} 14 | {"sentence": "expected from a college comedy that 's target audience has n't graduated from junior high school ", "label": 0, "idx": 27702} 15 | {"sentence": "is painfully bad ", "label": 0, "idx": 26245} 16 | {"sentence": "typical junkie opera ", "label": 0, "idx": 46531} 17 | {"sentence": "a timely look back at civil disobedience , anti-war movements and the power of strong voices . ", "label": 1, "idx": 64008} 18 | {"sentence": "any less entertaining ", "label": 1, "idx": 22287} 19 | {"sentence": "excellent performances ", "label": 1, "idx": 56366} 20 | {"sentence": "besides its terrific performances , is fontaine 's willingness to wander into the dark areas of parent-child relationships without flinching . ", "label": 1, "idx": 67134} 21 | {"sentence": "a fairly revealing study of its two main characters ", "label": 1, "idx": 470} 22 | {"sentence": "been made with an enormous amount of affection ", "label": 1, "idx": 42149} 23 | {"sentence": "strikes hardest ... when it reminds you how pertinent its dynamics remain ", "label": 1, "idx": 11613} 24 | {"sentence": "works well enough , since the thrills pop up frequently ", "label": 1, "idx": 46342} 25 | {"sentence": "be a good match of the sensibilities of two directors ", "label": 1, "idx": 13276} 26 | {"sentence": "for that matter , shrek ) ", "label": 1, "idx": 34861} 27 | {"sentence": "believable performances ", "label": 1, "idx": 62600} 28 | {"sentence": "has n't aged a day ", "label": 1, "idx": 38893} 29 | {"sentence": "compelling , provocative and prescient viewing ", "label": 1, "idx": 42721} 30 | {"sentence": "about long-lived friendships and the ways in which we all lose track of ourselves by trying ", "label": 1, "idx": 66909} 31 | {"sentence": "the hallmark of this bittersweet , uncommonly sincere movie that portrays the frank humanity of ... emotional recovery ", "label": 1, "idx": 65706} 32 | {"sentence": "padding ", "label": 1, "idx": 45968} 33 | -------------------------------------------------------------------------------- /data/few_shot/sst2/16_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"sentence": "all the boozy self-indulgence that brings out the worst in otherwise talented actors ", "label": 0, "idx": 49966} 2 | {"sentence": "exposes the limitations of his skill and the basic flaws in his vision ", "label": 0, "idx": 10533} 3 | {"sentence": "deserve better . ", "label": 0, "idx": 21415} 4 | {"sentence": "lacks the visual flair and bouncing bravado that characterizes better hip-hop clips and is content to recycle images and characters that were already tired 10 years ago . ", "label": 0, "idx": 59103} 5 | {"sentence": "morrissette has performed a difficult task indeed - he 's taken one of the world 's most fascinating stories and made it dull , lifeless , and irritating . ", "label": 0, "idx": 50300} 6 | {"sentence": "cheesy screenplay ", "label": 0, "idx": 52469} 7 | {"sentence": "inherent absurdity ", "label": 0, "idx": 50081} 8 | {"sentence": "it looks as though jay roach directed the film from the back of a taxicab ", "label": 0, "idx": 37073} 9 | {"sentence": ", but i believe a movie can be mindless without being the peak of all things insipid ", "label": 0, "idx": 2486} 10 | {"sentence": "restage the whole thing ", "label": 0, "idx": 32107} 11 | {"sentence": "while the third feels limited by its short running time ", "label": 0, "idx": 46601} 12 | {"sentence": "mindless and boring martial arts and gunplay with too little excitement and zero compelling storyline . ", "label": 0, "idx": 40360} 13 | {"sentence": "and altogether creepy ", "label": 0, "idx": 13115} 14 | {"sentence": "just the sort of lazy tearjerker that gives movies about ordinary folk a bad name . ", "label": 0, "idx": 32958} 15 | {"sentence": "phlegmatic bore ", "label": 0, "idx": 53105} 16 | {"sentence": "comedy death ", "label": 0, "idx": 63012} 17 | {"sentence": "a jarring , new-agey tone ", "label": 1, "idx": 49715} 18 | {"sentence": "the sum of its parts in today 's hollywood ", "label": 1, "idx": 37200} 19 | {"sentence": "is n't in this film , which may be why it works as well as it does . ", "label": 1, "idx": 21795} 20 | {"sentence": "bolstered by exceptional performances and a clear-eyed take on the economics of dealing and the pathology of ghetto fabulousness . ", "label": 1, "idx": 40957} 21 | {"sentence": "eerily ", "label": 1, "idx": 16382} 22 | {"sentence": "an eye on preserving a sense of mystery ", "label": 1, "idx": 26115} 23 | {"sentence": "lively songs ", "label": 1, "idx": 14778} 24 | {"sentence": "this fascinating look at israel in ferment feels as immediate as the latest news footage from gaza and , because of its heightened , well-shaped dramas , twice as powerful . ", "label": 1, "idx": 31844} 25 | {"sentence": "celebrated ", "label": 1, "idx": 30371} 26 | {"sentence": "the cast is phenomenal , especially the women . ", "label": 1, "idx": 46440} 27 | {"sentence": "the wit and hoopla ", "label": 1, "idx": 16917} 28 | {"sentence": "fulfilling ", "label": 1, "idx": 25543} 29 | {"sentence": "a fine production with splendid singing by angela gheorghiu , ruggero raimondi , and roberto alagna ", "label": 1, "idx": 54982} 30 | {"sentence": "is spooky and subtly in love with myth ", "label": 1, "idx": 52998} 31 | {"sentence": "decent performances ", "label": 1, "idx": 42795} 32 | {"sentence": "smart , savvy , compelling ", "label": 1, "idx": 56397} 33 | -------------------------------------------------------------------------------- /data/few_shot/sst2/16_shot/1_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"sentence": "edit ", "label": 0, "idx": 56301} 2 | {"sentence": "scarcely worth a mention apart from reporting on the number of tumbleweeds blowing through the empty theaters graced with its company . ", "label": 0, "idx": 65599} 3 | {"sentence": "( lin chung 's ) voice is rather unexceptional , even irritating ( at least to this western ear ) , making it awfully hard to buy the impetus for the complicated love triangle that develops between the three central characters . ", "label": 0, "idx": 57994} 4 | {"sentence": "an unintentionally surreal kid 's picture ... in which actors in bad bear suits enact a sort of inter-species parody of a vh1 behind the music episode ", "label": 0, "idx": 41778} 5 | {"sentence": "a terrible movie that some people will nevertheless find moving . ", "label": 0, "idx": 33437} 6 | {"sentence": "under my seat trying to sneak out of the theater ", "label": 0, "idx": 50994} 7 | {"sentence": "rice is too pedestrian a filmmaker to bring any edge or personality to the rising place that would set it apart from other deep south stories . ", "label": 0, "idx": 31091} 8 | {"sentence": "so insanely stupid , ", "label": 0, "idx": 26030} 9 | {"sentence": "to which the girls-behaving-badly film has fallen ", "label": 0, "idx": 32667} 10 | {"sentence": "like an oily arms dealer , squad car pile-ups and the requisite screaming captain ", "label": 0, "idx": 42455} 11 | {"sentence": "the movie dragged on ", "label": 0, "idx": 3077} 12 | {"sentence": "flimsy flicks ", "label": 0, "idx": 22799} 13 | {"sentence": "another , most of which involve precocious kids getting the better of obnoxious adults ", "label": 0, "idx": 22468} 14 | {"sentence": "exasperated ", "label": 0, "idx": 38334} 15 | {"sentence": "it 's just a movie that happens to have jackie chan in it . ", "label": 0, "idx": 36807} 16 | {"sentence": "be a one-trick pony ", "label": 0, "idx": 8400} 17 | {"sentence": "definitive account ", "label": 1, "idx": 56012} 18 | {"sentence": "greatest adventure ", "label": 1, "idx": 32141} 19 | {"sentence": "stamina and vitality ", "label": 1, "idx": 40985} 20 | {"sentence": "you believe the cast and crew thoroughly enjoyed themselves and believed in their small-budget film ", "label": 1, "idx": 65048} 21 | {"sentence": "with leonine power ", "label": 1, "idx": 1144} 22 | {"sentence": "for the best of hollywood 's comic-book ", "label": 1, "idx": 42381} 23 | {"sentence": "was immensely enjoyable thanks to great performances by both steve buscemi and rosario dawson ", "label": 1, "idx": 34960} 24 | {"sentence": "feel fresh ", "label": 1, "idx": 23180} 25 | {"sentence": "we do n't avert our eyes for a moment . ", "label": 1, "idx": 17044} 26 | {"sentence": "cute partnership ", "label": 1, "idx": 32600} 27 | {"sentence": "witty and often surprising ", "label": 1, "idx": 38474} 28 | {"sentence": "considerable talents ", "label": 1, "idx": 17698} 29 | {"sentence": "jolie gives it that extra little something that makes it worth checking out at theaters , especially if you 're in the mood for something more comfortable than challenging . ", "label": 1, "idx": 18158} 30 | {"sentence": "most opaque , ", "label": 1, "idx": 58364} 31 | {"sentence": "black hawk down with more heart ", "label": 1, "idx": 18958} 32 | {"sentence": "the wildly popular vin diesel in the equation ", "label": 1, "idx": 49443} 33 | -------------------------------------------------------------------------------- /data/few_shot/sst2/16_shot/32_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"sentence": "an unsuccessful attempt at a movie of ideas ", "label": 0, "idx": 11505} 2 | {"sentence": "as the filmmakers look down on their working-class subjects from their lofty perch , that finally makes sex with strangers , which opens today in the new york metropolitan area , so distasteful ", "label": 0, "idx": 18844} 3 | {"sentence": "feel like mopping up ", "label": 0, "idx": 2709} 4 | {"sentence": "an almost unbearably morbid ", "label": 0, "idx": 47075} 5 | {"sentence": "death , especially ", "label": 0, "idx": 2184} 6 | {"sentence": "humiliated by a pack of dogs who are smarter than him ", "label": 0, "idx": 19855} 7 | {"sentence": "the plot is paper-thin and the characters are n't interesting enough to watch them go about their daily activities for two whole hours . ", "label": 0, "idx": 46842} 8 | {"sentence": "but the project surrounding them is distressingly rote ", "label": 0, "idx": 56088} 9 | {"sentence": "such a mountain of clich\u00e9s and borrowed images ", "label": 0, "idx": 55220} 10 | {"sentence": "invites unflattering comparisons ", "label": 0, "idx": 23060} 11 | {"sentence": "impossible to claim that it is `` based on a true story '' ", "label": 0, "idx": 66476} 12 | {"sentence": "most excruciating 86 minutes ", "label": 0, "idx": 7936} 13 | {"sentence": "a bad sitcom ", "label": 0, "idx": 35371} 14 | {"sentence": "no psychology here , and no real narrative logic -- ", "label": 0, "idx": 18897} 15 | {"sentence": "by the incessant use of cell phones ", "label": 0, "idx": 11838} 16 | {"sentence": "overexposed ", "label": 0, "idx": 29393} 17 | {"sentence": "'s the very definition of epic adventure . ", "label": 1, "idx": 21139} 18 | {"sentence": "linger long after this film has ended ", "label": 1, "idx": 39406} 19 | {"sentence": "slyly exquisite ", "label": 1, "idx": 59267} 20 | {"sentence": ", this is grand-scale moviemaking for a larger-than-life figure , an artist who has been awarded mythic status in contemporary culture . ", "label": 1, "idx": 7276} 21 | {"sentence": "to the experience ", "label": 1, "idx": 3665} 22 | {"sentence": "a sincerely crafted picture ", "label": 1, "idx": 2354} 23 | {"sentence": "the worthy successor to a better tomorrow and ", "label": 1, "idx": 48633} 24 | {"sentence": "will be something to behold ", "label": 1, "idx": 42382} 25 | {"sentence": "are worthwhile . ", "label": 1, "idx": 24779} 26 | {"sentence": "be a breakthrough in filmmaking ", "label": 1, "idx": 43074} 27 | {"sentence": "aims to present an unflinching look at one man 's downfall , brought about by his lack of self-awareness ", "label": 1, "idx": 64674} 28 | {"sentence": "a rich tale of our times ", "label": 1, "idx": 10143} 29 | {"sentence": "if you 're willing to have fun with it , you wo n't feel cheated by the high infidelity of unfaithful . ", "label": 1, "idx": 16869} 30 | {"sentence": "the rare movie that 's as crisp and to the point as the novel on which it 's based . ", "label": 1, "idx": 12932} 31 | {"sentence": "that allows americans to finally revel in its splendor ", "label": 1, "idx": 44074} 32 | {"sentence": "two fine actors , ", "label": 1, "idx": 22265} 33 | -------------------------------------------------------------------------------- /data/few_shot/sst2/16_shot/42_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"sentence": "a dull , dumb and derivative horror ", "label": 0, "idx": 30440} 2 | {"sentence": "if george romero had directed this movie , it would n't have taken the protagonists a full hour to determine that in order to kill a zombie you must shoot it in the head ", "label": 0, "idx": 64285} 3 | {"sentence": "the acting is amateurish , the cinematography is atrocious ", "label": 0, "idx": 25294} 4 | {"sentence": "maudlin ending ", "label": 0, "idx": 63724} 5 | {"sentence": "derailed ", "label": 0, "idx": 60515} 6 | {"sentence": "what madonna does here ca n't properly be called acting -- more accurately , it 's moving and it 's talking and it 's occasionally gesturing , sometimes all at once ", "label": 0, "idx": 40570} 7 | {"sentence": "no worse a film ", "label": 0, "idx": 28619} 8 | {"sentence": "loosely connected characters and plots that never quite gel ", "label": 0, "idx": 35173} 9 | {"sentence": "we really have n't had a good cheesy b-movie playing in theaters since ... well ... since last week 's reign of fire ", "label": 0, "idx": 54341} 10 | {"sentence": "horrible '' and `` terrible ", "label": 0, "idx": 11698} 11 | {"sentence": "attal 's hang-ups surrounding infidelity are so old-fashioned and ", "label": 0, "idx": 39055} 12 | {"sentence": ", this cinderella story does n't have a single surprise up its sleeve . ", "label": 0, "idx": 6391} 13 | {"sentence": "'re not likely to have seen before , but beneath the exotic surface ( and exotic dancing ) it 's surprisingly old-fashioned . ", "label": 0, "idx": 34603} 14 | {"sentence": "all the movie 's narrative gymnastics ca n't disguise the fact that it 's inauthentic at its core and that its story just is n't worth telling . ", "label": 0, "idx": 63858} 15 | {"sentence": "hard to believe that a life like this can sound so dull ", "label": 0, "idx": 55750} 16 | {"sentence": "a rip-off of the matrix ", "label": 0, "idx": 58981} 17 | {"sentence": "acted meditation on both the profoundly devastating events of one year ago and the slow , painful healing process that has followed in their wake ", "label": 1, "idx": 41259} 18 | {"sentence": "this odd , poetic road movie , spiked by jolts of pop music , pretty much takes place in morton 's ever-watchful gaze -- and it 's a tribute to the actress , and to her inventive director , that the journey is such a mesmerizing one . ", "label": 1, "idx": 38498} 19 | {"sentence": "directed with purpose and finesse by england 's roger mitchell , who handily makes the move from pleasing , relatively lightweight commercial fare such as notting hill to commercial fare with real thematic heft . ", "label": 1, "idx": 528} 20 | {"sentence": "long and eventful ", "label": 1, "idx": 1334} 21 | {"sentence": "the best of the pierce brosnan james bond films to date . ", "label": 1, "idx": 41235} 22 | {"sentence": "babak payami 's boldly quirky iranian drama secret ballot ... a charming and evoking little ditty that manages to show the gentle and humane side of middle eastern world politics ", "label": 1, "idx": 14713} 23 | {"sentence": "a playful iranian parable ", "label": 1, "idx": 10807} 24 | {"sentence": "all too clever complexity ", "label": 1, "idx": 64917} 25 | {"sentence": ", stylized humor throughout ", "label": 1, "idx": 26412} 26 | {"sentence": "great team ", "label": 1, "idx": 26092} 27 | {"sentence": "the most multilayered and sympathetic female characters of the year ", "label": 1, "idx": 7078} 28 | {"sentence": "bear the laughter ", "label": 1, "idx": 37737} 29 | {"sentence": "a real winner -- smart , funny , subtle , and resonant ", "label": 1, "idx": 27349} 30 | {"sentence": "thoughtful and ", "label": 1, "idx": 6718} 31 | {"sentence": "has clever ways of capturing inner-city life during the reagan years ", "label": 1, "idx": 24915} 32 | {"sentence": "the film ascends to a kinetic life so teeming that even cranky adults may rediscover the quivering kid inside . ", "label": 1, "idx": 56349} 33 | -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/0_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"word": "take", "sentence1": "How should I take this message?", "sentence2": "He was often taken to be a man of means.", "start1": 13, "start2": 13, "end1": 17, "end2": 18, "idx": 2708, "label": 1} 2 | {"word": "touch", "sentence1": "This dish could use a touch of garlic.", "sentence2": "At his touch the room filled with lights.", "start1": 22, "start2": 7, "end1": 27, "end2": 12, "idx": 1625, "label": 0} 3 | {"word": "touch", "sentence1": "Suddenly, in the crowd, I felt a touch at my shoulder.", "sentence2": "This dish could use a touch of garlic.", "start1": 33, "start2": 22, "end1": 38, "end2": 27, "idx": 3158, "label": 0} 4 | {"word": "fill", "sentence1": "The mixer returned to the plant for another fill.", "sentence2": "He ate his fill of potatoes.", "start1": 44, "start2": 11, "end1": 48, "end2": 15, "idx": 2382, "label": 0} 5 | {"word": "rotation", "sentence1": "The medical resident finished a two-week rotation in pediatrics and began one in orthopaedics.", "sentence2": "Crop rotation makes a balanced demand on the fertility of the soil.", "start1": 41, "start2": 5, "end1": 49, "end2": 13, "idx": 142, "label": 1} 6 | {"word": "center", "sentence1": "His stories made him the center of the party.", "sentence2": "They had to reinforce the center.", "start1": 25, "start2": 26, "end1": 31, "end2": 32, "idx": 582, "label": 0} 7 | {"word": "lifesaving", "sentence1": "I'm normally a big fan of lifesaving, but in his case we can make an exception.", "sentence2": "He took a course in lifesaving.", "start1": 26, "start2": 20, "end1": 36, "end2": 30, "idx": 4510, "label": 1} 8 | {"word": "order", "sentence1": "IBM received an order for a hundred computers.", "sentence2": "The machine is now in working order.", "start1": 16, "start2": 30, "end1": 21, "end2": 35, "idx": 3689, "label": 0} 9 | {"word": "head", "sentence1": "200 head of cattle and 50 head of horses.", "sentence2": "At five years of age this head of cattle is worth perhaps $40.", "start1": 4, "start2": 26, "end1": 8, "end2": 30, "idx": 4601, "label": 1} 10 | {"word": "practice", "sentence1": "It is good practice to check each door and window before leaving.", "sentence2": "It is the usual practice of employees there to wear neckties only when meeting with customers.", "start1": 11, "start2": 16, "end1": 19, "end2": 24, "idx": 5325, "label": 1} 11 | {"word": "fall", "sentence1": "Fall into a strange way of thinking.", "sentence2": "Fall asleep.", "start1": 0, "start2": 0, "end1": 4, "end2": 4, "idx": 5192, "label": 1} 12 | {"word": "pull", "sentence1": "Pull one's horse to a stand.", "sentence2": "Pull the ball.", "start1": 0, "start2": 0, "end1": 4, "end2": 4, "idx": 5367, "label": 0} 13 | {"word": "make", "sentence1": "Clothes make the man.", "sentence2": "This makes the third infraction.", "start1": 8, "start2": 5, "end1": 12, "end2": 10, "idx": 15, "label": 1} 14 | {"word": "howl", "sentence1": "He gave a howl of pain.", "sentence2": "Howls of laughter.", "start1": 10, "start2": 0, "end1": 14, "end2": 5, "idx": 5370, "label": 1} 15 | {"word": "catch", "sentence1": "It sounds good but what's the catch?", "sentence2": "The player made an impressive catch.", "start1": 30, "start2": 30, "end1": 35, "end2": 35, "idx": 4034, "label": 0} 16 | {"word": "piece", "sentence1": "The children acted out a comic piece to amuse the guests.", "sentence2": "An important piece of the evidence.", "start1": 31, "start2": 13, "end1": 36, "end2": 18, "idx": 3824, "label": 0} 17 | {"word": "product", "sentence1": "We've got to sell a lot of product by the end of the month.", "sentence2": "The product of 2 and 3 is 6.", "start1": 27, "start2": 4, "end1": 34, "end2": 11, "idx": 3068, "label": 0} 18 | {"word": "head", "sentence1": "12 head of big cattle and 14 head of branded calves.", "sentence2": "They shot 20 head of quail.", "start1": 3, "start2": 13, "end1": 7, "end2": 17, "idx": 4127, "label": 1} 19 | {"word": "fire", "sentence1": "Fire a bullet.", "sentence2": "Surrender fires the cold skepticism.", "start1": 0, "start2": 10, "end1": 4, "end2": 15, "idx": 2012, "label": 0} 20 | {"word": "case", "sentence1": "It may rain in which case the picnic will be canceled.", "sentence2": "A door case.", "start1": 21, "start2": 7, "end1": 25, "end2": 11, "idx": 2267, "label": 0} 21 | {"word": "stay", "sentence1": "I can only stay for an hour.", "sentence2": "The judge stayed the execution order.", "start1": 11, "start2": 10, "end1": 15, "end2": 16, "idx": 4606, "label": 0} 22 | {"word": "education", "sentence1": "He received no formal education.", "sentence2": "A good teacher is essential for a good education.", "start1": 22, "start2": 39, "end1": 31, "end2": 48, "idx": 1393, "label": 1} 23 | {"word": "performance", "sentence1": "They admired his performance under stress.", "sentence2": "They listened to ten different performances.", "start1": 17, "start2": 31, "end1": 28, "end2": 43, "idx": 3461, "label": 0} 24 | {"word": "land", "sentence1": "The plane is about to land.", "sentence2": "The ship landed in Pearl Harbor.", "start1": 22, "start2": 9, "end1": 26, "end2": 15, "idx": 728, "label": 1} 25 | {"word": "hollow", "sentence1": "The hollow of the hand or of a tree.", "sentence2": "He built himself a cabin in a hollow high up in the Rockies.", "start1": 4, "start2": 30, "end1": 10, "end2": 36, "idx": 2884, "label": 0} 26 | {"word": "scene", "sentence1": "The most moving scene is the final one, where he realizes he has wasted his whole life.", "sentence2": "There were some very erotic scenes in the movie, although it was not classified as pornography.", "start1": 16, "start2": 28, "end1": 21, "end2": 34, "idx": 2255, "label": 1} 27 | {"word": "have", "sentence1": "I have two years left.", "sentence2": "He had an insulin shock after eating three candy bars.", "start1": 2, "start2": 3, "end1": 6, "end2": 6, "idx": 3267, "label": 0} 28 | {"word": "dig", "sentence1": "Dig the oars into the water.", "sentence2": "Dig your foot into the floor.", "start1": 0, "start2": 0, "end1": 3, "end2": 3, "idx": 2922, "label": 1} 29 | {"word": "empathize", "sentence1": "You really have to empathize with their doing it.", "sentence2": "You really have to empathize with them.", "start1": 19, "start2": 19, "end1": 28, "end2": 28, "idx": 4785, "label": 1} 30 | {"word": "normalize", "sentence1": "There is little hope that the two countries will normalize relations; their governments seem to hate each other and would just as soon stay on bad terms.", "sentence2": "Normalize the temperature.", "start1": 49, "start2": 0, "end1": 58, "end2": 9, "idx": 5031, "label": 1} 31 | {"word": "fix", "sentence1": "Fix breakfast for the guests, please.", "sentence2": "That'll fix him good!", "start1": 0, "start2": 8, "end1": 3, "end2": 11, "idx": 2806, "label": 0} 32 | {"word": "see", "sentence1": "What message do you see in this letter?", "sentence2": "We had to see a psychiatrist.", "start1": 20, "start2": 10, "end1": 23, "end2": 13, "idx": 1516, "label": 0} 33 | -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/100_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"word": "write", "sentence1": "Please write to me every week.", "sentence2": "Your son has been writing on the wall.", "start1": 7, "start2": 18, "end1": 12, "end2": 25, "idx": 310, "label": 0} 2 | {"word": "fire", "sentence1": "Hold your fire until you can see the whites of their eyes.", "sentence2": "Barbecue over an open fire.", "start1": 10, "start2": 22, "end1": 14, "end2": 26, "idx": 5188, "label": 0} 3 | {"word": "make", "sentence1": "Make a mess in one's office.", "sentence2": "She'll make a fine president.", "start1": 0, "start2": 7, "end1": 4, "end2": 11, "idx": 4970, "label": 0} 4 | {"word": "place", "sentence1": "He comes from a place near Chicago.", "sentence2": "He lost his place on the page.", "start1": 16, "start2": 12, "end1": 21, "end2": 17, "idx": 5082, "label": 0} 5 | {"word": "anger", "sentence1": "You anger too easily.", "sentence2": "He angers easily.", "start1": 4, "start2": 3, "end1": 9, "end2": 9, "idx": 4613, "label": 1} 6 | {"word": "yes", "sentence1": "I was hoping for a yes.", "sentence2": "Was that a yes?", "start1": 19, "start2": 11, "end1": 22, "end2": 14, "idx": 3778, "label": 1} 7 | {"word": "property", "sentence1": "Charm is his most endearing property.", "sentence2": "A study of the physical properties of atomic particles.", "start1": 28, "start2": 24, "end1": 36, "end2": 34, "idx": 3706, "label": 1} 8 | {"word": "block", "sentence1": "Block a nerve.", "sentence2": "Block a garment.", "start1": 0, "start2": 0, "end1": 5, "end2": 5, "idx": 780, "label": 0} 9 | {"word": "buy", "sentence1": "I'm going to buy my father something nice for his birthday.", "sentence2": "This sum will buy you a ride on the train.", "start1": 13, "start2": 14, "end1": 16, "end2": 17, "idx": 4894, "label": 0} 10 | {"word": "see", "sentence1": "I want to see whether she speaks French.", "sentence2": "I went to see my friend Mary the other day.", "start1": 10, "start2": 10, "end1": 13, "end2": 13, "idx": 3797, "label": 0} 11 | {"word": "eyeball", "sentence1": "A good cook can often just eyeball the correct quantities of ingredients.", "sentence2": "Each geometric construction must be exact; eyeballing it and getting close does not count.", "start1": 27, "start2": 43, "end1": 34, "end2": 53, "idx": 616, "label": 1} 12 | {"word": "pull", "sentence1": "Pull a horse.", "sentence2": "Pull weeds.", "start1": 0, "start2": 0, "end1": 4, "end2": 4, "idx": 4276, "label": 0} 13 | {"word": "pull", "sentence1": "Pull a horse.", "sentence2": "You'll be sent home if you pull another stunt like that.", "start1": 0, "start2": 27, "end1": 4, "end2": 31, "idx": 269, "label": 0} 14 | {"word": "day", "sentence1": "He deserves his day in court.", "sentence2": "In the days of sailing ships.", "start1": 16, "start2": 7, "end1": 19, "end2": 11, "idx": 2376, "label": 0} 15 | {"word": "pack", "sentence1": "You had better pack your swollen ankle with ice.", "sentence2": "Pack the faucet.", "start1": 15, "start2": 0, "end1": 19, "end2": 4, "idx": 339, "label": 0} 16 | {"word": "rotation", "sentence1": "The manager had only four starting pitchers in his rotation.", "sentence2": "Crop rotation makes a balanced demand on the fertility of the soil.", "start1": 51, "start2": 5, "end1": 59, "end2": 13, "idx": 2109, "label": 1} 17 | {"word": "shortness", "sentence1": "Can cause shortness of breath.", "sentence2": "There was no shortness of money.", "start1": 10, "start2": 13, "end1": 19, "end2": 22, "idx": 4011, "label": 1} 18 | {"word": "voice", "sentence1": "The noisy voice of the waterfall.", "sentence2": "The incessant voices of the artillery.", "start1": 10, "start2": 14, "end1": 15, "end2": 20, "idx": 4330, "label": 1} 19 | {"word": "begin", "sentence1": "You can not even begin to understand the problem we had to deal with during the war.", "sentence2": "He began early in the day.", "start1": 17, "start2": 3, "end1": 22, "end2": 8, "idx": 750, "label": 0} 20 | {"word": "go", "sentence1": "Messages must go through diplomatic channels.", "sentence2": "We should go farther in this matter.", "start1": 14, "start2": 10, "end1": 16, "end2": 12, "idx": 4284, "label": 1} 21 | {"word": "keep", "sentence1": "These potatoes keep for a long time.", "sentence2": "She keeps an apartment in Paris for her shopping trips.", "start1": 15, "start2": 4, "end1": 19, "end2": 9, "idx": 2504, "label": 0} 22 | {"word": "turn", "sentence1": "Turn the heat down.", "sentence2": "This milk has turned; it smells awful.", "start1": 0, "start2": 14, "end1": 4, "end2": 20, "idx": 1151, "label": 0} 23 | {"word": "face", "sentence1": "He washed his face.", "sentence2": "She has a pretty face.", "start1": 14, "start2": 17, "end1": 18, "end2": 21, "idx": 995, "label": 1} 24 | {"word": "fire", "sentence1": "There was a fire at the school last night and the whole place burned down.", "sentence2": "We sat around the fire singing songs and telling stories.", "start1": 12, "start2": 18, "end1": 16, "end2": 22, "idx": 4167, "label": 0} 25 | {"word": "chain", "sentence1": "The third link of the silver chain needs to be resoldered.", "sentence2": "This led to an unfortunate chain of events.", "start1": 29, "start2": 27, "end1": 34, "end2": 32, "idx": 4406, "label": 1} 26 | {"word": "charge", "sentence1": "Can I charge this purchase?", "sentence2": "The suspect was charged with murdering his wife.", "start1": 6, "start2": 16, "end1": 12, "end2": 23, "idx": 5416, "label": 0} 27 | {"word": "make", "sentence1": "Make one's way into the forest.", "sentence2": "Make yourself clear.", "start1": 0, "start2": 0, "end1": 4, "end2": 4, "idx": 2955, "label": 0} 28 | {"word": "pass", "sentence1": "He had a pass for three days.", "sentence2": "It was not possible to complete the computation in a single pass.", "start1": 9, "start2": 60, "end1": 13, "end2": 64, "idx": 959, "label": 0} 29 | {"word": "patch", "sentence1": "The world economy had a rough patch in the 1930s.", "sentence2": "The storms last summer washed away parts of the road so we can expect some rough patches up ahead.", "start1": 30, "start2": 81, "end1": 35, "end2": 88, "idx": 5212, "label": 1} 30 | {"word": "nomination", "sentence1": "The Republican nomination for Governor.", "sentence2": "His nomination was hotly protested.", "start1": 15, "start2": 4, "end1": 25, "end2": 14, "idx": 2506, "label": 0} 31 | {"word": "system", "sentence1": "A system generating hydrogen peroxide.", "sentence2": "In a static system oil can not be replaced by water on a surface.", "start1": 2, "start2": 12, "end1": 8, "end2": 18, "idx": 4069, "label": 1} 32 | {"word": "academy", "sentence1": "The military academy at West Point.", "sentence2": "Academies of literature and philology.", "start1": 13, "start2": 0, "end1": 20, "end2": 9, "idx": 4487, "label": 0} 33 | -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"word": "order", "sentence1": "Order ruled in the streets.", "sentence2": "The machinery is out of order.", "start1": 0, "start2": 24, "end1": 5, "end2": 29, "idx": 4909, "label": 1} 2 | {"word": "take", "sentence1": "Take the gun from your pocket.", "sentence2": "The test grader takes off a point for every misspelled word.", "start1": 0, "start2": 16, "end1": 4, "end2": 21, "idx": 1744, "label": 1} 3 | {"word": "wash", "sentence1": "The car is so dirty, we need to wash it.", "sentence2": "The nurse washed away the blood.", "start1": 32, "start2": 10, "end1": 36, "end2": 16, "idx": 2968, "label": 1} 4 | {"word": "permit", "sentence1": "This will permit the rain to run off.", "sentence2": "This permits the water to rush in.", "start1": 10, "start2": 5, "end1": 16, "end2": 12, "idx": 1352, "label": 1} 5 | {"word": "fall", "sentence1": "Fall by the wayside.", "sentence2": "Christmas falls on a Monday this year.", "start1": 0, "start2": 10, "end1": 4, "end2": 15, "idx": 273, "label": 0} 6 | {"word": "process", "sentence1": "This product of last month's quality standards committee is quite good, even though the process was flawed.", "sentence2": "The process of calcification begins later for boys than for girls.", "start1": 88, "start2": 4, "end1": 95, "end2": 11, "idx": 3660, "label": 1} 7 | {"word": "raise", "sentence1": "To raise Sandy Hook light.", "sentence2": "Raise your hands.", "start1": 3, "start2": 0, "end1": 8, "end2": 5, "idx": 4173, "label": 1} 8 | {"word": "sense", "sentence1": "A sense of self.", "sentence2": "A sense of security.", "start1": 2, "start2": 2, "end1": 7, "end2": 7, "idx": 4561, "label": 1} 9 | {"word": "draw", "sentence1": "I draw a line here.", "sentence2": "Draw parallels.", "start1": 2, "start2": 0, "end1": 6, "end2": 4, "idx": 4338, "label": 1} 10 | {"word": "translate", "sentence1": "Is there a need to translate the psychiatrist's remarks?", "sentence2": "Can you translate the instructions in this manual for a layman?", "start1": 19, "start2": 8, "end1": 28, "end2": 17, "idx": 3025, "label": 1} 11 | {"word": "feel", "sentence1": "I can feel the sadness in his poems.", "sentence2": "She felt an object brushing her arm.", "start1": 6, "start2": 4, "end1": 10, "end2": 8, "idx": 4691, "label": 0} 12 | {"word": "floor", "sentence1": "The pit floor showed where a ring of post holes had been.", "sentence2": "The floor of the pelvis.", "start1": 8, "start2": 4, "end1": 13, "end2": 9, "idx": 5263, "label": 1} 13 | {"word": "shot", "sentence1": "His parting shot was 'drop dead'.", "sentence2": "A good shot requires good balance and tempo.", "start1": 12, "start2": 7, "end1": 16, "end2": 11, "idx": 5323, "label": 0} 14 | {"word": "land", "sentence1": "The plane is about to land.", "sentence2": "The ship landed in Pearl Harbor.", "start1": 22, "start2": 9, "end1": 26, "end2": 15, "idx": 728, "label": 1} 15 | {"word": "mean", "sentence1": "I mean no harm.", "sentence2": "I didn't mean to knock your tooth out.", "start1": 2, "start2": 9, "end1": 6, "end2": 13, "idx": 3856, "label": 1} 16 | {"word": "fatigue", "sentence1": "Political fatigue.", "sentence2": "The American public is experiencing scandal fatigue.", "start1": 10, "start2": 44, "end1": 17, "end2": 51, "idx": 2428, "label": 1} 17 | {"word": "curmudgeon", "sentence1": "There's a cranky curmudgeon working at the hospital who gives all the patients and other doctors flak.", "sentence2": "John Doe's old age and stubborn aversion to new ideas make him a curmudgeon of a candidate.", "start1": 17, "start2": 65, "end1": 27, "end2": 75, "idx": 971, "label": 1} 18 | {"word": "trojan", "sentence1": "The contents of a trojan can be a virus or a worm.", "sentence2": "When he downloaded the free game it turned out to be a trojan.", "start1": 18, "start2": 55, "end1": 24, "end2": 61, "idx": 1045, "label": 1} 19 | {"word": "business", "sentence1": "Show me where the business was today.", "sentence2": "Government and business could not agree.", "start1": 18, "start2": 15, "end1": 26, "end2": 23, "idx": 4111, "label": 0} 20 | {"word": "air", "sentence1": "Air pollution.", "sentence2": "There was a tension in the air which made me suspect an approaching storm.", "start1": 0, "start2": 27, "end1": 3, "end2": 30, "idx": 3536, "label": 1} 21 | {"word": "match", "sentence1": "His interests didn't match her interests.", "sentence2": "Let them match their best athletes against ours.", "start1": 21, "start2": 9, "end1": 26, "end2": 14, "idx": 1669, "label": 0} 22 | {"word": "twinkle", "sentence1": "There's a perpetual twinkle in his eyes.", "sentence2": "He was a rotund, jolly man with a twinkle in his eye.", "start1": 20, "start2": 34, "end1": 27, "end2": 41, "idx": 1651, "label": 1} 23 | {"word": "crack", "sentence1": "To crack a whip.", "sentence2": "Registrations cracked through the 30,000 mark in the county.", "start1": 3, "start2": 14, "end1": 8, "end2": 21, "idx": 1273, "label": 0} 24 | {"word": "light", "sentence1": "Can you shed light on this problem?", "sentence2": "Put that light out!", "start1": 13, "start2": 9, "end1": 18, "end2": 14, "idx": 1831, "label": 0} 25 | {"word": "connect", "sentence1": "The new railroad will connect the northern part of the state to the southern part.", "sentence2": "Herman connected the computers.", "start1": 22, "start2": 7, "end1": 29, "end2": 16, "idx": 304, "label": 1} 26 | {"word": "taproot", "sentence1": "The taproot of his resentment.", "sentence2": "Genius and insanity spring from the same taproot.", "start1": 4, "start2": 41, "end1": 11, "end2": 48, "idx": 3820, "label": 1} 27 | {"word": "dip", "sentence1": "Dip the brush into the paint.", "sentence2": "I had to dip into my savings to buy him this present.", "start1": 0, "start2": 9, "end1": 3, "end2": 12, "idx": 5395, "label": 0} 28 | {"word": "business", "sentence1": "Show me where the business was today.", "sentence2": "Business has been slow lately.", "start1": 18, "start2": 0, "end1": 26, "end2": 8, "idx": 4660, "label": 1} 29 | {"word": "death", "sentence1": "He seemed more content in death than he had ever been in life.", "sentence2": "Her death came as a terrible shock.", "start1": 26, "start2": 4, "end1": 31, "end2": 9, "idx": 315, "label": 0} 30 | {"word": "cycle", "sentence1": "The \"Ring of the Nibelung\" is a cycle of four operas by Richard Wagner, the famous nineteenth-century German composer.", "sentence2": "A year constitutes a cycle of the seasons.", "start1": 32, "start2": 21, "end1": 37, "end2": 26, "idx": 2301, "label": 0} 31 | {"word": "chance", "sentence1": "You take a chance when you let her drive.", "sentence2": "We had the chance to meet the president last week.", "start1": 11, "start2": 11, "end1": 17, "end2": 17, "idx": 1079, "label": 0} 32 | {"word": "extension", "sentence1": "Most BASIC files use the filename extension .BAS.", "sentence2": "The extension of 'satellite of Mars' is the set containing only Demos and Phobos.", "start1": 34, "start2": 4, "end1": 43, "end2": 13, "idx": 3628, "label": 0} 33 | -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/1_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"word": "light", "sentence1": "Hey, buddy, you got a light?", "sentence2": "Can you shed light on this problem?", "start1": 22, "start2": 13, "end1": 27, "end2": 18, "idx": 763, "label": 0} 2 | {"word": "interest", "sentence1": "In the interest of safety.", "sentence2": "Victorian furniture is an interest of mine.", "start1": 7, "start2": 26, "end1": 15, "end2": 34, "idx": 116, "label": 0} 3 | {"word": "vermin", "sentence1": "Boys in the village have probably been shooting vermin.", "sentence2": "Cereals must be protected from mice and other vermin.", "start1": 48, "start2": 46, "end1": 54, "end2": 52, "idx": 4244, "label": 1} 4 | {"word": "grain", "sentence1": "Sand of a fine grain.", "sentence2": "A stone of coarse grain.", "start1": 15, "start2": 18, "end1": 20, "end2": 23, "idx": 5233, "label": 1} 5 | {"word": "bear", "sentence1": "Bear a heavy load.", "sentence2": "Bear a resemblance.", "start1": 0, "start2": 0, "end1": 4, "end2": 4, "idx": 4497, "label": 0} 6 | {"word": "cover", "sentence1": "Cover her face with a handkerchief.", "sentence2": "This should cover everyone in the group.", "start1": 0, "start2": 12, "end1": 5, "end2": 17, "idx": 2064, "label": 0} 7 | {"word": "flood", "sentence1": "Flood the market with tennis shoes.", "sentence2": "The images flooded his mind.", "start1": 0, "start2": 11, "end1": 5, "end2": 18, "idx": 5132, "label": 0} 8 | {"word": "drag", "sentence1": "He performed in drag.", "sentence2": "He went to the party dressed in drag.", "start1": 16, "start2": 32, "end1": 20, "end2": 36, "idx": 2217, "label": 1} 9 | {"word": "echo", "sentence1": "Napoleon III was an echo of the mighty Emperor but an infinitely better man.", "sentence2": "His contention contains more than an echo of Rousseau.", "start1": 20, "start2": 37, "end1": 24, "end2": 41, "idx": 1162, "label": 1} 10 | {"word": "wealth", "sentence1": "She brings a wealth of knowledge to the project.", "sentence2": "Great wealth is not a sign of great intelligence.", "start1": 13, "start2": 6, "end1": 19, "end2": 12, "idx": 3687, "label": 1} 11 | {"word": "glimpse", "sentence1": "From the window he could catch a glimpse of the lake.", "sentence2": "I only got a glimpse of the car, so I can tell you the colour but not the registration number.", "start1": 33, "start2": 13, "end1": 40, "end2": 20, "idx": 4215, "label": 0} 12 | {"word": "head", "sentence1": "200 head of cattle.", "sentence2": "A reduction in the assessment per head of sheep.", "start1": 4, "start2": 34, "end1": 8, "end2": 38, "idx": 1956, "label": 1} 13 | {"word": "head", "sentence1": "I've got to go to the head.", "sentence2": "This song keeps going through my head.", "start1": 22, "start2": 33, "end1": 26, "end2": 37, "idx": 2837, "label": 0} 14 | {"word": "carry", "sentence1": "These bonds carry warrants.", "sentence2": "We carry a very large mortgage.", "start1": 12, "start2": 3, "end1": 17, "end2": 8, "idx": 3446, "label": 0} 15 | {"word": "pay", "sentence1": "She had to pay the penalty for speaking out rashly.", "sentence2": "He paid her off the books and in kind where possible.", "start1": 11, "start2": 3, "end1": 14, "end2": 7, "idx": 4887, "label": 0} 16 | {"word": "drive", "sentence1": "His drive and energy exhausted his co-workers.", "sentence2": "The riverside drive offers many exciting scenic views.", "start1": 4, "start2": 14, "end1": 9, "end2": 19, "idx": 4447, "label": 0} 17 | {"word": "make", "sentence1": "What make of car is that?", "sentence2": "What make of car do you drive?", "start1": 5, "start2": 5, "end1": 9, "end2": 9, "idx": 4262, "label": 1} 18 | {"word": "forbid", "sentence1": "The rules forbid smoking.", "sentence2": "The rules forbid our smoking.", "start1": 10, "start2": 10, "end1": 16, "end2": 16, "idx": 4145, "label": 1} 19 | {"word": "series", "sentence1": "A comedy series.", "sentence2": "His coin collection included the complete series of Indian-head pennies.", "start1": 9, "start2": 42, "end1": 15, "end2": 48, "idx": 3892, "label": 0} 20 | {"word": "dibs", "sentence1": "Who's got dibs on the chips?", "sentence2": "Dibs means I get the hammock.", "start1": 10, "start2": 0, "end1": 14, "end2": 4, "idx": 1375, "label": 1} 21 | {"word": "feel", "sentence1": "I feel that he doesn't like me.", "sentence2": "I felt my way cautiously through the dangerous business maneuver.", "start1": 2, "start2": 2, "end1": 6, "end2": 6, "idx": 2704, "label": 0} 22 | {"word": "work", "sentence1": "I will work hard to improve my grades.", "sentence2": "They worked on the task.", "start1": 7, "start2": 5, "end1": 11, "end2": 11, "idx": 1225, "label": 1} 23 | {"word": "make", "sentence1": "Make an effort.", "sentence2": "Make love, not war.", "start1": 0, "start2": 0, "end1": 4, "end2": 4, "idx": 4930, "label": 1} 24 | {"word": "make", "sentence1": "One swallow does not a summer make.", "sentence2": "This makes the third infraction.", "start1": 30, "start2": 5, "end1": 34, "end2": 10, "idx": 3176, "label": 1} 25 | {"word": "piece", "sentence1": "I need to rest for a piece.", "sentence2": "She bought a lovely piece of china.", "start1": 21, "start2": 20, "end1": 26, "end2": 25, "idx": 847, "label": 0} 26 | {"word": "shoot", "sentence1": "Shoot craps.", "sentence2": "Shoot a movie.", "start1": 0, "start2": 0, "end1": 5, "end2": 5, "idx": 3724, "label": 0} 27 | {"word": "marriage", "sentence1": "Pat and Leslie's marriage to each other lasted forty years.", "sentence2": "You should enter marriage for love.", "start1": 17, "start2": 17, "end1": 25, "end2": 25, "idx": 3618, "label": 0} 28 | {"word": "anchorage", "sentence1": "The plant needs a firm anchorage.", "sentence2": "The mother provides emotional anchorage for the entire family.", "start1": 23, "start2": 30, "end1": 32, "end2": 39, "idx": 4226, "label": 1} 29 | {"word": "war", "sentence1": "The \"war on drugs\" is a campaign against the use of narcotic drugs.", "sentence2": "The war on poverty.", "start1": 5, "start2": 4, "end1": 8, "end2": 7, "idx": 3272, "label": 1} 30 | {"word": "turn", "sentence1": "Turn a key.", "sentence2": "Turn the clay on the wheel.", "start1": 0, "start2": 0, "end1": 4, "end2": 4, "idx": 3693, "label": 0} 31 | {"word": "go", "sentence1": "Go about the world in a certain manner.", "sentence2": "His knowledge doesn't go very far.", "start1": 0, "start2": 22, "end1": 2, "end2": 24, "idx": 2850, "label": 0} 32 | {"word": "cot", "sentence1": "A roller cot (the clothing of a drawing roller in a spinning frame).", "sentence2": "A cot for a sore finger.", "start1": 9, "start2": 2, "end1": 12, "end2": 5, "idx": 580, "label": 1} 33 | -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/32_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"word": "see", "sentence1": "You should see a lawyer.", "sentence2": "We had to see a psychiatrist.", "start1": 11, "start2": 10, "end1": 14, "end2": 13, "idx": 96, "label": 1} 2 | {"word": "turn", "sentence1": "Turn your face to the wall.", "sentence2": "She is turning 50 this year.", "start1": 0, "start2": 7, "end1": 4, "end2": 14, "idx": 1388, "label": 0} 3 | {"word": "give", "sentence1": "What will you give her for her birthday?", "sentence2": "She gave a pair of shoes to her husband for their anniversary.", "start1": 14, "start2": 4, "end1": 18, "end2": 8, "idx": 3963, "label": 1} 4 | {"word": "call", "sentence1": "The mayor likes to call on some of the prominent citizens.", "sentence2": "She called her children lazy and ungrateful.", "start1": 19, "start2": 4, "end1": 23, "end2": 10, "idx": 2890, "label": 0} 5 | {"word": "embody", "sentence1": "As the car salesman approached, wearing a plaid suit and slicked-back hair, he seemed to embody sleaze.", "sentence2": "This painting embodies the feelings of the Romantic period.", "start1": 89, "start2": 14, "end1": 95, "end2": 22, "idx": 1848, "label": 1} 6 | {"word": "class", "sentence1": "The class of all sets is not a set.", "sentence2": "There are two classes of detergents.", "start1": 4, "start2": 14, "end1": 9, "end2": 21, "idx": 5094, "label": 1} 7 | {"word": "sign", "sentence1": "The hunters found deer sign at the end of the trail.", "sentence2": "Those clouds show signs of raining soon.", "start1": 23, "start2": 18, "end1": 27, "end2": 23, "idx": 2760, "label": 1} 8 | {"word": "par", "sentence1": "Par for this course is 72.", "sentence2": "He needs to make this shot for par.", "start1": 0, "start2": 31, "end1": 3, "end2": 34, "idx": 2206, "label": 1} 9 | {"word": "patch", "sentence1": "A cabbage patch.", "sentence2": "A briar patch.", "start1": 10, "start2": 8, "end1": 15, "end2": 13, "idx": 1368, "label": 1} 10 | {"word": "lift", "sentence1": "Lift a ban.", "sentence2": "I lifted the books onto the table.", "start1": 0, "start2": 2, "end1": 4, "end2": 8, "idx": 5390, "label": 0} 11 | {"word": "bring", "sentence1": "Waiter, please bring me a single malt whiskey.", "sentence2": "This brings me to the main point.", "start1": 15, "start2": 5, "end1": 20, "end2": 11, "idx": 4379, "label": 1} 12 | {"word": "shock", "sentence1": "Whole fields of wheat in shock.", "sentence2": "It came as a shock to learn that he was injured.", "start1": 25, "start2": 13, "end1": 30, "end2": 18, "idx": 699, "label": 0} 13 | {"word": "stuff", "sentence1": "There's good stuff in that book.", "sentence2": "The stuff of heros.", "start1": 13, "start2": 4, "end1": 18, "end2": 9, "idx": 1533, "label": 0} 14 | {"word": "undergo", "sentence1": "Undergo a strange sensation.", "sentence2": "The chemical undergoes a sudden change.", "start1": 0, "start2": 13, "end1": 7, "end2": 22, "idx": 3838, "label": 1} 15 | {"word": "refine", "sentence1": "Refine paper stock.", "sentence2": "Refine pig iron.", "start1": 0, "start2": 0, "end1": 6, "end2": 6, "idx": 2986, "label": 1} 16 | {"word": "carry", "sentence1": "These bonds carry warrants.", "sentence2": "The loan carries a high interest rate.", "start1": 12, "start2": 9, "end1": 17, "end2": 16, "idx": 556, "label": 1} 17 | {"word": "sentence", "sentence1": "The judge leniently granted a sentence with no hard time. He is not living at home because he is doing time.", "sentence2": "His sentence was 5 to 10 years.", "start1": 30, "start2": 4, "end1": 38, "end2": 12, "idx": 2867, "label": 1} 18 | {"word": "get", "sentence1": "She didn't get his name when they met the first time.", "sentence2": "The ice storm got my hibiscus.", "start1": 11, "start2": 14, "end1": 14, "end2": 17, "idx": 2964, "label": 0} 19 | {"word": "tramp", "sentence1": "A homeless tramp.", "sentence2": "The tramp of military boots.", "start1": 11, "start2": 4, "end1": 16, "end2": 9, "idx": 1147, "label": 0} 20 | {"word": "crush", "sentence1": "Crush an aluminum can.", "sentence2": "She was crushed by his refusal of her invitation.", "start1": 0, "start2": 8, "end1": 5, "end2": 15, "idx": 1662, "label": 0} 21 | {"word": "application", "sentence1": "The application of indexes to tables of data.", "sentence2": "The application of maximum thrust.", "start1": 4, "start2": 4, "end1": 15, "end2": 15, "idx": 2442, "label": 1} 22 | {"word": "wall", "sentence1": "The wall followed the road.", "sentence2": "Stomach walls.", "start1": 4, "start2": 8, "end1": 8, "end2": 13, "idx": 1156, "label": 0} 23 | {"word": "answer", "sentence1": "Answer the riddle.", "sentence2": "Answer a question.", "start1": 0, "start2": 0, "end1": 6, "end2": 6, "idx": 465, "label": 1} 24 | {"word": "pay", "sentence1": "She had to pay the penalty for speaking out rashly.", "sentence2": "You'll pay for this opinion later.", "start1": 11, "start2": 7, "end1": 14, "end2": 10, "idx": 4563, "label": 1} 25 | {"word": "use", "sentence1": "I can't use this tool.", "sentence2": "Use the plastic bags to store the food.", "start1": 8, "start2": 0, "end1": 11, "end2": 3, "idx": 2072, "label": 1} 26 | {"word": "work", "sentence1": "This dough does not work easily.", "sentence2": "Work the phones.", "start1": 20, "start2": 0, "end1": 24, "end2": 4, "idx": 1598, "label": 0} 27 | {"word": "make", "sentence1": "Make a mistake.", "sentence2": "My wife made me buy a new sofa.", "start1": 0, "start2": 8, "end1": 4, "end2": 12, "idx": 4180, "label": 0} 28 | {"word": "mean", "sentence1": "The sky is red this morning -- does that mean we're in for a storm?", "sentence2": "I didn't mean to knock your tooth out.", "start1": 41, "start2": 9, "end1": 45, "end2": 13, "idx": 2162, "label": 0} 29 | {"word": "day", "sentence1": "My day began early this morning.", "sentence2": "It was a busy day on the stock exchange.", "start1": 3, "start2": 14, "end1": 6, "end2": 17, "idx": 1443, "label": 1} 30 | {"word": "leak", "sentence1": "A leak in a roof.", "sentence2": "He had to clean up the leak.", "start1": 2, "start2": 23, "end1": 6, "end2": 27, "idx": 1189, "label": 1} 31 | {"word": "par", "sentence1": "Par for this course is 72.", "sentence2": "On a par with the best.", "start1": 0, "start2": 5, "end1": 3, "end2": 8, "idx": 2825, "label": 0} 32 | {"word": "authority", "sentence1": "She lost all her respect and authority after turning up drunk to the meeting.", "sentence2": "Tennessee Valley Authority.", "start1": 29, "start2": 17, "end1": 38, "end2": 26, "idx": 5381, "label": 0} 33 | -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/42_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"word": "tell", "sentence1": "He could tell that she was unhappy.", "sentence2": "She told him how he did it.", "start1": 9, "start2": 4, "end1": 13, "end2": 8, "idx": 79, "label": 0} 2 | {"word": "coordinate", "sentence1": "Coordinate our efforts.", "sentence2": "Coordinate his actions with that of his colleagues.", "start1": 0, "start2": 0, "end1": 10, "end2": 10, "idx": 3161, "label": 1} 3 | {"word": "cut", "sentence1": "That's our finest cut of meat.", "sentence2": "His cut in the lining revealed the hidden jewels.", "start1": 18, "start2": 4, "end1": 21, "end2": 7, "idx": 1643, "label": 0} 4 | {"word": "come", "sentence1": "Nothing good will come of this.", "sentence2": "Come down here!", "start1": 18, "start2": 0, "end1": 22, "end2": 4, "idx": 167, "label": 0} 5 | {"word": "focus", "sentence1": "If you're going to beat your competitors, you need to focus.", "sentence2": "We focused on what he wanted.", "start1": 54, "start2": 3, "end1": 59, "end2": 10, "idx": 439, "label": 1} 6 | {"word": "slack", "sentence1": "Suddenly they were in a slack and the water was motionless.", "sentence2": "A gradual slack in output.", "start1": 24, "start2": 10, "end1": 29, "end2": 15, "idx": 2531, "label": 0} 7 | {"word": "go", "sentence1": "How fast does your new car go?", "sentence2": "This door goes to the basement.", "start1": 27, "start2": 10, "end1": 29, "end2": 14, "idx": 1421, "label": 0} 8 | {"word": "rate", "sentence1": "The rate of production at the factory is skyrocketing.", "sentence2": "He works at a great rate.", "start1": 4, "start2": 20, "end1": 8, "end2": 24, "idx": 2834, "label": 1} 9 | {"word": "keep", "sentence1": "May God keep you.", "sentence2": "She keeps an apartment in Paris for her shopping trips.", "start1": 8, "start2": 4, "end1": 12, "end2": 9, "idx": 4357, "label": 0} 10 | {"word": "keep", "sentence1": "Keep food for a week in the pantry.", "sentence2": "She keeps a sixpack and a week's worth of supplies in the refrigerator.", "start1": 0, "start2": 4, "end1": 4, "end2": 9, "idx": 168, "label": 1} 11 | {"word": "pay", "sentence1": "It will pay to wait.", "sentence2": "He paid me 5000 pounds for the car.", "start1": 8, "start2": 3, "end1": 11, "end2": 7, "idx": 5421, "label": 0} 12 | {"word": "hold", "sentence1": "Hold one's nose.", "sentence2": "I hold with those who say life is sacred.", "start1": 0, "start2": 2, "end1": 4, "end2": 6, "idx": 4581, "label": 0} 13 | {"word": "coverage", "sentence1": "The enemy fire is increasing -- can we get some immediate coverage from those bunkers?", "sentence2": "There are overlapping coverages on your insurance policies.", "start1": 58, "start2": 22, "end1": 66, "end2": 31, "idx": 4081, "label": 1} 14 | {"word": "defense", "sentence1": "In defense he said the other man started it.", "sentence2": "The artillery battered down the defenses.", "start1": 3, "start2": 32, "end1": 10, "end2": 40, "idx": 1971, "label": 0} 15 | {"word": "approach", "sentence1": "To approach the city.", "sentence2": "He was an admirable poet, and thought even to have approached Homer. -- Sir William Temple.", "start1": 3, "start2": 51, "end1": 11, "end2": 61, "idx": 2629, "label": 1} 16 | {"word": "catch", "sentence1": "Catch the mood.", "sentence2": "The Rolls Royce caught us near the exit ramp.", "start1": 0, "start2": 16, "end1": 5, "end2": 22, "idx": 2481, "label": 0} 17 | {"word": "business", "sentence1": "They did nearly a million dollars of business over the long weekend.", "sentence2": "Business has been slow lately.", "start1": 37, "start2": 0, "end1": 45, "end2": 8, "idx": 3889, "label": 1} 18 | {"word": "trace", "sentence1": "Trace a design.", "sentence2": "The women traced the pasture.", "start1": 0, "start2": 10, "end1": 5, "end2": 16, "idx": 3988, "label": 0} 19 | {"word": "gob", "sentence1": "He's always stuffing his gob with fast food.", "sentence2": "She's got such a gob on her -- she's always gossiping about someone or other.", "start1": 25, "start2": 17, "end1": 28, "end2": 20, "idx": 4349, "label": 1} 20 | {"word": "life", "sentence1": "He lived a long and happy life.", "sentence2": "Life depends on many chemical and physical processes.", "start1": 26, "start2": 0, "end1": 30, "end2": 4, "idx": 800, "label": 0} 21 | {"word": "acknowledgment", "sentence1": "She must have seen him but she gave no sign of acknowledgment.", "sentence2": "The preface contained an acknowledgment of those who had helped her.", "start1": 47, "start2": 25, "end1": 61, "end2": 39, "idx": 811, "label": 1} 22 | {"word": "meet", "sentence1": "The company agrees to meet the cost of any repairs.", "sentence2": "Does this paper meet the requirements for the degree?", "start1": 22, "start2": 16, "end1": 26, "end2": 20, "idx": 3075, "label": 1} 23 | {"word": "grace", "sentence1": "It was debated whether saving grace could be obtained outside the membership of the church.", "sentence2": "I'm so grateful to God for the grace that He has given me.", "start1": 30, "start2": 31, "end1": 35, "end2": 36, "idx": 5310, "label": 0} 24 | {"word": "occasion", "sentence1": "An occasion arranged to honor the president.", "sentence2": "There was no occasion for complaint.", "start1": 3, "start2": 13, "end1": 11, "end2": 21, "idx": 5306, "label": 0} 25 | {"word": "difference", "sentence1": "The difference in her is amazing.", "sentence2": "It just won't make much of a difference to anyone.", "start1": 4, "start2": 29, "end1": 14, "end2": 39, "idx": 2862, "label": 1} 26 | {"word": "blow", "sentence1": "Certain insects are said to blow.", "sentence2": "There's nothing more thrilling to the whale watcher than to see a whale surface and blow.", "start1": 28, "start2": 84, "end1": 32, "end2": 88, "idx": 3114, "label": 0} 27 | {"word": "be", "sentence1": "I may be an hour.", "sentence2": "It has been three years since my grandmother died. (similar to My grandmother died three years ago, but emphasizes the intervening period).", "start1": 6, "start2": 7, "end1": 8, "end2": 11, "idx": 2602, "label": 0} 28 | {"word": "side", "sentence1": "He is on the purchasing side of the business.", "sentence2": "Look on the bright side.", "start1": 24, "start2": 19, "end1": 28, "end2": 23, "idx": 230, "label": 1} 29 | {"word": "ferment", "sentence1": "We ferment the grapes for a very long time to achieve high alcohol content.", "sentence2": "The Middle East is fermenting.", "start1": 3, "start2": 19, "end1": 10, "end2": 29, "idx": 3053, "label": 0} 30 | {"word": "concord", "sentence1": "Both philosophers concord on this point.", "sentence2": "Concord the conditions for the marriage of the Prince of Wales with a commoner.", "start1": 18, "start2": 0, "end1": 25, "end2": 7, "idx": 4803, "label": 0} 31 | {"word": "support", "sentence1": "They gave him emotional support during difficult times.", "sentence2": "His support kept the family together.", "start1": 24, "start2": 4, "end1": 31, "end2": 11, "idx": 1769, "label": 1} 32 | {"word": "way", "sentence1": "Make way for.", "sentence2": "Didn't know the way home.", "start1": 5, "start2": 16, "end1": 8, "end2": 19, "idx": 3787, "label": 0} 33 | -------------------------------------------------------------------------------- /data/few_shot/wsc/32_shot/0_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"text": "Fred is the only man still alive who remembers my great-grandfather . He was a remarkable man.", "span1_index": 0, "span2_index": 12, "span1_text": "Fred", "span2_text": "He", "idx": 85, "label": 0} 2 | {"text": "Thomson visited Cooper 's grave in 1765. At that date he had been travelling for five years.", "span1_index": 2, "span2_index": 10, "span1_text": "Cooper", "span2_text": "he", "idx": 132, "label": 0} 3 | {"text": "Then he took a piece of iron the shape of the horse horse's foot , and clapped it on, and drove some nails through the shoe quite into its hoof, so that the shoe was firmly on", "span1_index": 12, "span2_index": 17, "span1_text": "horse's foot", "span2_text": "it", "idx": 96, "label": 0} 4 | {"text": "Mary tucked her daughter Anne into bed, so that she could work.", "span1_index": 0, "span2_index": 9, "span1_text": "Mary", "span2_text": "she", "idx": 107, "label": 1} 5 | {"text": "Dr. Adams informed Kate that she had retired and presented several options for future treatment.", "span1_index": 0, "span2_index": 5, "span1_text": "Dr. Adams", "span2_text": "she", "idx": 140, "label": 1} 6 | {"text": "Frank was upset with Tom because the toaster he had bought from him didn't work.", "span1_index": 4, "span2_index": 8, "span1_text": "Tom", "span2_text": "he", "idx": 37, "label": 0} 7 | {"text": "I sat there feeling rather like a chappie I'd once read about in a book , who murdered another cove and hid the body under the dining-room table , and then had to be the life and soul of a dinner party, with it there all the time.", "span1_index": 27, "span2_index": 43, "span1_text": "table", "span2_text": "it", "idx": 481, "label": 0} 8 | {"text": "Billy cried because Toby wouldn't accept his toy.", "span1_index": 0, "span2_index": 6, "span1_text": "Billy", "span2_text": "his", "idx": 90, "label": 1} 9 | {"text": "Steve follows Fred 's example in everything. He admires him hugely.", "span1_index": 0, "span2_index": 7, "span1_text": "Steve", "span2_text": "He", "idx": 514, "label": 1} 10 | {"text": "Steve follows Fred 's example in everything. He influences him hugely.", "span1_index": 0, "span2_index": 7, "span1_text": "Steve", "span2_text": "He", "idx": 235, "label": 0} 11 | {"text": "The sack of potatoes had been placed below the bag of flour , so it had to be moved first.", "span1_index": 8, "span2_index": 14, "span1_text": "the bag of flour", "span2_text": "it", "idx": 172, "label": 1} 12 | {"text": "The mothers of Arthur and Celeste have come to the town to fetch them. They are very happy to have them back, but they scold them just the same because they ran away.", "span1_index": 3, "span2_index": 20, "span1_text": "Arthur and Celeste", "span2_text": "them", "idx": 440, "label": 1} 13 | {"text": "Sam's drawing was hung just above Tina's and it did look much better with another one above it.", "span1_index": 0, "span2_index": 8, "span1_text": "Sam's drawing", "span2_text": "it", "idx": 66, "label": 0} 14 | {"text": "Babar wonders how he can get new clothing. Luckily, a very rich old man who has always been fond of little elephants understands right away that he is longing for a fine suit. As he likes to make people happy, he gives him his wallet.", "span1_index": 12, "span2_index": 42, "span1_text": "old man", "span2_text": "him", "idx": 494, "label": 0} 15 | {"text": "This book introduced Shakespeare to Goethe ; it was a fine selection of his writing.", "span1_index": 5, "span2_index": 13, "span1_text": "Goethe", "span2_text": "his", "idx": 493, "label": 0} 16 | {"text": "The table was piled high with food , and on the floor beside it there were crocks, baskets, and a five-quart pail of milk.", "span1_index": 1, "span2_index": 13, "span1_text": "table", "span2_text": "it", "idx": 337, "label": 1} 17 | {"text": "I sallied out for a bit of food , more to pass the time than because I wanted it .", "span1_index": 7, "span2_index": 18, "span1_text": "food", "span2_text": "it", "idx": 553, "label": 1} 18 | {"text": "The donkey wished a wart on its hind leg would disappear, and it did.", "span1_index": 8, "span2_index": 12, "span1_text": "leg", "span2_text": "it", "idx": 346, "label": 0} 19 | {"text": "The politicians far away in Washington could not know the settlers so they must make rules to regulate them.", "span1_index": 1, "span2_index": 12, "span1_text": "politicians", "span2_text": "they", "idx": 175, "label": 1} 20 | {"text": "Carol believed that Rebecca suspected that she had stolen the watch.", "span1_index": 3, "span2_index": 6, "span1_text": "Rebecca", "span2_text": "she", "idx": 477, "label": 0} 21 | {"text": "Susan knows all about Ann 's personal problems because she is indiscreet.", "span1_index": 4, "span2_index": 9, "span1_text": "Ann", "span2_text": "she", "idx": 261, "label": 1} 22 | {"text": "Jane knocked on Susan 's door, but there was no answer. She was disappointed.", "span1_index": 0, "span2_index": 11, "span1_text": "Jane", "span2_text": "She", "idx": 428, "label": 1} 23 | {"text": "Emma Emma's mother had died long ago, and her education had been managed by an excellent woman as governess.", "span1_index": 0, "span2_index": 8, "span1_text": "Emma", "span2_text": "her", "idx": 435, "label": 1} 24 | {"text": "Emma did not pass the ball to Janie although she saw that she was open.", "span1_index": 7, "span2_index": 9, "span1_text": "Janie", "span2_text": "she", "idx": 282, "label": 0} 25 | {"text": "The Wainwrights treated Mr. Crowley like a prince until he made his will in their favor; then they treated him like dirt. Folks said he died just to be rid of their everlasting nagging.", "span1_index": 22, "span2_index": 31, "span1_text": "Folks", "span2_text": "their", "idx": 118, "label": 0} 26 | {"text": "Mama came over and sat down beside Alice . Gently she stroked her hair and let the child weep.", "span1_index": 0, "span2_index": 12, "span1_text": "Mama", "span2_text": "her", "idx": 350, "label": 0} 27 | {"text": "John couldn't see the stage with Billy in front of him because he is so short.", "span1_index": 6, "span2_index": 12, "span1_text": "Billy", "span2_text": "he", "idx": 531, "label": 0} 28 | {"text": "Babar wonders how he can get new clothing. Luckily, a very rich old man who has always been fond of little elephants understands right away that he is longing for a fine suit. As he likes to make people happy, he gives him his wallet.", "span1_index": 0, "span2_index": 43, "span1_text": "Babar", "span2_text": "his", "idx": 142, "label": 0} 29 | {"text": "Jim signaled the barman and gestured toward his empty glass", "span1_index": 2, "span2_index": 7, "span1_text": "the barman", "span2_text": "his", "idx": 165, "label": 0} 30 | {"text": "Mama came over and sat down beside Alice . Gently she stroked her hair and let the child weep.", "span1_index": 0, "span2_index": 10, "span1_text": "Mama", "span2_text": "she", "idx": 491, "label": 1} 31 | {"text": "Sam took French classes from Adam , because he was eager to speak it fluently.", "span1_index": 5, "span2_index": 8, "span1_text": "Adam", "span2_text": "he", "idx": 153, "label": 0} 32 | {"text": "John promised Bill to leave, so an hour later he left.", "span1_index": 0, "span2_index": 9, "span1_text": "John", "span2_text": "he", "idx": 247, "label": 1} 33 | -------------------------------------------------------------------------------- /data/few_shot/wsc/32_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"text": "Sergeant Holmes asked the girls to describe the intruder . Nancy not only provided the policeman with an excellent description of the heavyset thirty-year-old prowler, but drew a rough sketch of his face.", "span1_index": 7, "span2_index": 31, "span1_text": "the intruder", "span2_text": "his", "idx": 159, "label": 1} 2 | {"text": "Our mistress hired a room for my crippled sister Nelly , and she gave her knitting and needlework when she was able to do it, and when she was ill she sent her dinners and many nice comfortable things and was like a mother to her.", "span1_index": 9, "span2_index": 32, "span1_text": "Nelly", "span2_text": "her", "idx": 209, "label": 1} 3 | {"text": "Do you suppose that Peter is responsible for the captain \u2019s illness? Maybe he bribed the cook to put something in his food.", "span1_index": 4, "span2_index": 21, "span1_text": "Peter", "span2_text": "his", "idx": 63, "label": 0} 4 | {"text": "Tom said \"Check\" to Ralph as he took his bishop.", "span1_index": 0, "span2_index": 8, "span1_text": "Tom", "span2_text": "his", "idx": 377, "label": 0} 5 | {"text": "Jackson was greatly influenced by Arnold , though he lived two centuries later.", "span1_index": 0, "span2_index": 8, "span1_text": "Jackson", "span2_text": "he", "idx": 315, "label": 1} 6 | {"text": "Steve follows Fred 's example in everything. He influences him hugely.", "span1_index": 2, "span2_index": 7, "span1_text": "Fred", "span2_text": "He", "idx": 262, "label": 1} 7 | {"text": "When Tommy dropped his ice cream, Timmy giggled, so father gave him a stern look.", "span1_index": 1, "span2_index": 11, "span1_text": "Tommy", "span2_text": "him", "idx": 201, "label": 0} 8 | {"text": "The user changed his password from \" GrWQWu8JyC \" to \" willow-towered Canopy Huntertropic wrestles \" as it was easy to forget.", "span1_index": 7, "span2_index": 17, "span1_text": "GrWQWu8JyC", "span2_text": "it", "idx": 495, "label": 1} 9 | {"text": "Susan knew that Ann 's son had been in a car accident, so she told her about it.", "span1_index": 0, "span2_index": 13, "span1_text": "Susan", "span2_text": "she", "idx": 321, "label": 1} 10 | {"text": "Carol believed that Rebecca suspected that she had stolen the watch.", "span1_index": 0, "span2_index": 6, "span1_text": "Carol", "span2_text": "she", "idx": 294, "label": 1} 11 | {"text": "Mark heard Steve 's feet going down the ladder. The door of the shop closed after him . He ran to look out the window.", "span1_index": 0, "span2_index": 16, "span1_text": "Mark", "span2_text": "him", "idx": 396, "label": 0} 12 | {"text": "Beth didn't get angry with Sally , who had cut her off, because she stopped and apologized.", "span1_index": 5, "span2_index": 13, "span1_text": "Sally", "span2_text": "she", "idx": 515, "label": 1} 13 | {"text": "When Tatyana reached the cabin, her mother was sleeping. She was careful not to disturb her , undressing and climbing back into her berth.", "span1_index": 6, "span2_index": 5, "span1_text": "mother", "span2_text": "her", "idx": 392, "label": 1} 14 | {"text": "Emma did not pass the ball to Janie although she was open.", "span1_index": 7, "span2_index": 9, "span1_text": "Janie", "span2_text": "she", "idx": 10, "label": 1} 15 | {"text": "I asked Dave to get me my sweater from the other side of the yacht . While he was gone, I rested my arm on the rail over there and suddenly it gave way.", "span1_index": 26, "span2_index": 31, "span1_text": "rail", "span2_text": "it", "idx": 266, "label": 1} 16 | {"text": "John hired Bill to take care of him .", "span1_index": 2, "span2_index": 7, "span1_text": "Bill", "span2_text": "him", "idx": 53, "label": 0} 17 | {"text": "The user changed his password from \" GrWQWu8JyC \" to \" willow-towered Canopy Huntertropic wrestles \" as it was easy to remember.", "span1_index": 7, "span2_index": 17, "span1_text": "GrWQWu8JyC", "span2_text": "it", "idx": 309, "label": 0} 18 | {"text": "The table was piled high with food , and on the floor beside it there were crocks, baskets, and a five-quart pail of milk.", "span1_index": 11, "span2_index": 13, "span1_text": "floor", "span2_text": "it", "idx": 60, "label": 0} 19 | {"text": "Jane knocked on Susan 's door but she did not get an answer.", "span1_index": 0, "span2_index": 7, "span1_text": "Jane", "span2_text": "she", "idx": 443, "label": 1} 20 | {"text": "I poured water from the bottle into the cup until it was full.", "span1_index": 4, "span2_index": 10, "span1_text": "the bottle", "span2_text": "it", "idx": 51, "label": 0} 21 | {"text": "Sir Clifford wants me to find him a new groom , about twenty or twenty-one, who knows his business. His old coachman is getting feeble, and he wants a man to work with him and get into his ways, who would be able, when the old man was pensioned off, to step into his place", "span1_index": 20, "span2_index": 53, "span1_text": "old coachman", "span2_text": "his", "idx": 34, "label": 1} 22 | {"text": "The sculpture rolled off the shelf because it wasn't level.", "span1_index": 0, "span2_index": 7, "span1_text": "The sculpture", "span2_text": "it", "idx": 507, "label": 0} 23 | {"text": "Thomson visited Cooper 's grave in 1765. At that date he had been dead for five years.", "span1_index": 2, "span2_index": 10, "span1_text": "Cooper", "span2_text": "he", "idx": 338, "label": 1} 24 | {"text": "Although they ran at about the same speed, Sue beat Sally because she had such a bad start.", "span1_index": 8, "span2_index": 12, "span1_text": "Sue", "span2_text": "she", "idx": 349, "label": 0} 25 | {"text": "Jane knocked on Susan 's door, but there was no answer. She was disappointed.", "span1_index": 0, "span2_index": 11, "span1_text": "Jane", "span2_text": "She", "idx": 428, "label": 1} 26 | {"text": "John ordered Bill to leave, so an hour later he left.", "span1_index": 0, "span2_index": 9, "span1_text": "John", "span2_text": "he", "idx": 391, "label": 0} 27 | {"text": "As Andrea in the crop duster passed over Susan , she could see the landing gear.", "span1_index": 8, "span2_index": 10, "span1_text": "Susan", "span2_text": "she", "idx": 414, "label": 1} 28 | {"text": "Babar wonders how he can get new clothing. Luckily, a very rich old man who has always been fond of little elephants understands right away that he is longing for a fine suit. As he likes to make people happy, he gives him his wallet.", "span1_index": 0, "span2_index": 40, "span1_text": "Babar", "span2_text": "he", "idx": 246, "label": 0} 29 | {"text": "The mothers of Arthur and Celeste have come to the town to fetch them. They are very happy to have them back, but they scold them just the same because they ran away.", "span1_index": 3, "span2_index": 30, "span1_text": "Arthur and Celeste", "span2_text": "they", "idx": 411, "label": 1} 30 | {"text": "I asked Dave to get me my sweater from the other side of the yacht . While he was gone, I rested my arm on the rail over there and suddenly it gave way.", "span1_index": 7, "span2_index": 31, "span1_text": "sweater", "span2_text": "it", "idx": 310, "label": 0} 31 | {"text": "Carol believed that Rebecca suspected that she had stolen the watch.", "span1_index": 3, "span2_index": 6, "span1_text": "Rebecca", "span2_text": "she", "idx": 477, "label": 0} 32 | {"text": "John couldn't see the stage with Billy in front of him because he is so tall.", "span1_index": 0, "span2_index": 12, "span1_text": "John", "span2_text": "he", "idx": 258, "label": 0} 33 | -------------------------------------------------------------------------------- /data/few_shot/wsc/32_shot/1_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"text": "So Mark slept. It was daylight when he woke with Warren 's hand upon his shoulder.", "span1_index": 1, "span2_index": 14, "span1_text": "Mark", "span2_text": "his", "idx": 292, "label": 1} 2 | {"text": "Sir Clifford wants me to find him a new groom , about twenty or twenty-one, who knows his business. His old coachman is getting feeble, and he wants a man to work with him and get into his ways, who would be able, when the old man was pensioned off, to step into his place", "span1_index": 0, "span2_index": 53, "span1_text": "Sir Clifford", "span2_text": "his", "idx": 474, "label": 0} 3 | {"text": "The table won't fit through the doorway because it is too wide.", "span1_index": 0, "span2_index": 8, "span1_text": "The table", "span2_text": "it", "idx": 67, "label": 1} 4 | {"text": "The delivery truck zoomed by the school bus because it was going so slow.", "span1_index": 5, "span2_index": 9, "span1_text": "the school bus", "span2_text": "it", "idx": 133, "label": 1} 5 | {"text": "Although they ran at about the same speed, Sue beat Sally because she had such a bad start.", "span1_index": 8, "span2_index": 12, "span1_text": "Sue", "span2_text": "she", "idx": 349, "label": 0} 6 | {"text": "Paul tried to call George on the phone, but he wasn't available.", "span1_index": 0, "span2_index": 9, "span1_text": "Paul", "span2_text": "he", "idx": 361, "label": 0} 7 | {"text": "The foxes are getting in at night and attacking the chickens . They have gotten very nervous.", "span1_index": 9, "span2_index": 12, "span1_text": "the chickens", "span2_text": "They", "idx": 160, "label": 1} 8 | {"text": "Bob paid for Charlie 's college education, but now Charlie acts as though it never happened. He is very ungrateful.", "span1_index": 3, "span2_index": 16, "span1_text": "Charlie", "span2_text": "He", "idx": 117, "label": 1} 9 | {"text": "The fish ate the worm . It was tasty.", "span1_index": 3, "span2_index": 6, "span1_text": "the worm", "span2_text": "It", "idx": 29, "label": 1} 10 | {"text": "Babar wonders how he can get new clothing. Luckily, a very rich old man who has always been fond of little elephants understands right away that he is longing for a fine suit. As he likes to make people happy, he gives him his wallet.", "span1_index": 12, "span2_index": 43, "span1_text": "old man", "span2_text": "his", "idx": 404, "label": 1} 11 | {"text": "Fred is the only man alive who still remembers my father as an infant. When Fred first saw my father , he was twelve years old.", "span1_index": 0, "span2_index": 21, "span1_text": "Fred", "span2_text": "he", "idx": 304, "label": 1} 12 | {"text": "Although they ran at about the same speed, Sue beat Sally because she had such a good start.", "span1_index": 8, "span2_index": 12, "span1_text": "Sue", "span2_text": "she", "idx": 402, "label": 1} 13 | {"text": "During a game of tag, Ethan chased Luke because he was \"it\".", "span1_index": 5, "span2_index": 9, "span1_text": "Ethan", "span2_text": "he", "idx": 119, "label": 1} 14 | {"text": "Paul tried to call George on the phone, but he wasn't available.", "span1_index": 4, "span2_index": 9, "span1_text": "George", "span2_text": "he", "idx": 65, "label": 1} 15 | {"text": "Kirilov ceded the presidency to Shatov because he was more popular.", "span1_index": 0, "span2_index": 7, "span1_text": "Kirilov", "span2_text": "he", "idx": 351, "label": 0} 16 | {"text": "Jim signaled the barman and gestured toward his empty glass", "span1_index": 2, "span2_index": 7, "span1_text": "the barman", "span2_text": "his", "idx": 165, "label": 0} 17 | {"text": "The sack of potatoes had been placed below the bag of flour , so it had to be moved first.", "span1_index": 8, "span2_index": 14, "span1_text": "the bag of flour", "span2_text": "it", "idx": 172, "label": 1} 18 | {"text": "Tatyana knew that Grandma always enjoyed serving an abundance of food to her guests. Now Tatyana watched as Grandma gathered Tatyana 's small mother into a wide, scrawny embrace and then propelled her to the table, lifting her shawl from her shoulders, seating her in the place of honor, and saying simply: \"There's plenty.\"", "span1_index": 0, "span2_index": 12, "span1_text": "Tatyana", "span2_text": "her", "idx": 180, "label": 0} 19 | {"text": "John hired him self out to Bill to take care of him .", "span1_index": 0, "span2_index": 2, "span1_text": "John", "span2_text": "him", "idx": 293, "label": 0} 20 | {"text": "At the Loebner competition the judges couldn't figure out which respondents were the chatbots because they were so advanced.", "span1_index": 4, "span2_index": 15, "span1_text": "the judges", "span2_text": "they", "idx": 157, "label": 0} 21 | {"text": "It is not easy to space buttonholes exactly the same distance apart, and it is very difficult to cut them precisely the right size . The tiniest slip of the scissors will make the hole too large, and even one thread uncut will leave it too small.", "span1_index": 29, "span2_index": 44, "span1_text": "the scissors", "span2_text": "it", "idx": 429, "label": 0} 22 | {"text": "Babar wonders how he can get new clothing. Luckily, a very rich old man who has always been fond of little elephants understands right away that he is longing for a fine suit. As he likes to make people happy, he gives him his wallet.", "span1_index": 12, "span2_index": 42, "span1_text": "old man", "span2_text": "him", "idx": 494, "label": 0} 23 | {"text": "As Andrea in the crop duster passed over Susan , she could see the landing gear.", "span1_index": 1, "span2_index": 10, "span1_text": "Andrea", "span2_text": "she", "idx": 444, "label": 0} 24 | {"text": "There is a pillar between me and the stage , and I can't see around it .", "span1_index": 7, "span2_index": 15, "span1_text": "the stage", "span2_text": "it", "idx": 525, "label": 0} 25 | {"text": "Jackson was greatly influenced by Arnold , though he lived two centuries earlier.", "span1_index": 5, "span2_index": 8, "span1_text": "Arnold", "span2_text": "he", "idx": 492, "label": 1} 26 | {"text": "Mark told Pete many lies about himself, which Pete included in his book. He should have been more skeptical.", "span1_index": 0, "span2_index": 13, "span1_text": "Mark", "span2_text": "He", "idx": 0, "label": 0} 27 | {"text": "The donkey wished a wart on its hind leg would disappear, and it did.", "span1_index": 8, "span2_index": 12, "span1_text": "leg", "span2_text": "it", "idx": 346, "label": 0} 28 | {"text": "The woman held the girl against her will.", "span1_index": 0, "span2_index": 6, "span1_text": "The woman", "span2_text": "her", "idx": 498, "label": 0} 29 | {"text": "There is a pillar between me and the stage , and I can't see it .", "span1_index": 7, "span2_index": 14, "span1_text": "the stage", "span2_text": "it", "idx": 450, "label": 1} 30 | {"text": "What about the time you cut up tulip bulbs in the hamburgers because you thought they were onions?", "span1_index": 7, "span2_index": 15, "span1_text": "tulip bulbs", "span2_text": "they", "idx": 223, "label": 1} 31 | {"text": "The mothers of Arthur and Celeste have come to the town to fetch them. They are very happy to have them back, but they scold them just the same because they ran away.", "span1_index": 1, "span2_index": 20, "span1_text": "mothers", "span2_text": "them", "idx": 500, "label": 0} 32 | {"text": "We went to the lake , because a shark had been seen at the ocean beach , so it was a safer place to swim.", "span1_index": 13, "span2_index": 18, "span1_text": "the ocean beach", "span2_text": "it", "idx": 146, "label": 0} 33 | -------------------------------------------------------------------------------- /data/few_shot/wsc/32_shot/32_seed.jsonl: -------------------------------------------------------------------------------- 1 | {"text": "Frank was upset with Tom because the toaster he had bought from him didn't work.", "span1_index": 4, "span2_index": 8, "span1_text": "Tom", "span2_text": "he", "idx": 37, "label": 0} 2 | {"text": "Jim signaled the barman and gestured toward his bathroom key.", "span1_index": 0, "span2_index": 7, "span1_text": "Jim", "span2_text": "his", "idx": 449, "label": 0} 3 | {"text": "The sack of potatoes had been placed below the bag of flour , so it had to be moved first.", "span1_index": 0, "span2_index": 14, "span1_text": "The sack of potatoes", "span2_text": "it", "idx": 526, "label": 0} 4 | {"text": "The city councilmen refused the demonstrators a permit because they advocated violence.", "span1_index": 4, "span2_index": 9, "span1_text": "the demonstrators", "span2_text": "they", "idx": 16, "label": 1} 5 | {"text": "Equally swoon-worthy is C.K. Dexter Haven , a pallid young dandy holding a jade-handled walking stick, with a poodle asleep at his feet", "span1_index": 5, "span2_index": 21, "span1_text": "Haven", "span2_text": "his", "idx": 40, "label": 1} 6 | {"text": "Joe paid the detective after he received the final report on the case.", "span1_index": 0, "span2_index": 5, "span1_text": "Joe", "span2_text": "he", "idx": 441, "label": 1} 7 | {"text": "In July, Kamtchatka declared war on Yakutsk . Since Yakutsk 's army was much better equipped and ten times larger, they were defeated within weeks.", "span1_index": 6, "span2_index": 20, "span1_text": "Yakutsk", "span2_text": "they", "idx": 532, "label": 0} 8 | {"text": "The table won't fit through the doorway because it is too wide.", "span1_index": 5, "span2_index": 8, "span1_text": "the doorway", "span2_text": "it", "idx": 383, "label": 0} 9 | {"text": "The cat was lying by the mouse hole waiting for the mouse , but it was too cautious.", "span1_index": 5, "span2_index": 14, "span1_text": "the mouse", "span2_text": "it", "idx": 547, "label": 1} 10 | {"text": "The sack of potatoes had been placed below the bag of flour , so it had to be moved first.", "span1_index": 8, "span2_index": 14, "span1_text": "the bag of flour", "span2_text": "it", "idx": 172, "label": 1} 11 | {"text": "When Tommy dropped his ice cream, Timmy giggled, so father gave him a sympathetic look.", "span1_index": 6, "span2_index": 11, "span1_text": "Timmy", "span2_text": "him", "idx": 17, "label": 0} 12 | {"text": "The journalists interviewed the stars of the new movie. They were very cooperative, so the interview lasted for a long time.", "span1_index": 0, "span2_index": 9, "span1_text": "The journalists", "span2_text": "They", "idx": 19, "label": 0} 13 | {"text": "The table was piled high with food , and on the floor beside it there were crocks, baskets, and a five-quart pail of milk.", "span1_index": 11, "span2_index": 13, "span1_text": "floor", "span2_text": "it", "idx": 60, "label": 0} 14 | {"text": "The boy continued to whip the pony , and eventually the pony threw him over. John laughed out quite loud. \"Good for him,\" he said. ", "span1_index": 15, "span2_index": 22, "span1_text": "John", "span2_text": "him", "idx": 453, "label": 0} 15 | {"text": "Jane knocked on Susan 's door, but there was no answer. She was out.", "span1_index": 3, "span2_index": 11, "span1_text": "Susan", "span2_text": "She", "idx": 378, "label": 1} 16 | {"text": "The police arrested all of the gang members . They were trying to stop the drug trade in the neighborhood.", "span1_index": 0, "span2_index": 9, "span1_text": "The police", "span2_text": "They", "idx": 193, "label": 1} 17 | {"text": "Frank felt vindicated when his longtime rival Bill revealed that he was the winner of the competition.", "span1_index": 0, "span2_index": 10, "span1_text": "Frank", "span2_text": "he", "idx": 225, "label": 1} 18 | {"text": "Beth didn't get angry with Sally , who had cut her off, because she stopped and apologized.", "span1_index": 5, "span2_index": 13, "span1_text": "Sally", "span2_text": "she", "idx": 515, "label": 1} 19 | {"text": "We went to the lake , because a shark had been seen at the ocean beach , so it was a dangerous place to swim.", "span1_index": 3, "span2_index": 18, "span1_text": "the lake", "span2_text": "it", "idx": 365, "label": 0} 20 | {"text": "Archaeologists have concluded that humans lived in Laputa 20,000 years ago. They hunted for evidence on the river banks.", "span1_index": 0, "span2_index": 11, "span1_text": "Archaeologists", "span2_text": "They", "idx": 372, "label": 1} 21 | {"text": "It is not easy to space buttonholes exactly the same distance apart, and it is very difficult to cut them precisely the right size . The tiniest slip of the scissors will make the hole too large, and even one thread uncut will leave it too small.", "span1_index": 21, "span2_index": 44, "span1_text": "the right size", "span2_text": "it", "idx": 527, "label": 0} 22 | {"text": "Jane knocked on Susan 's door, but there was no answer. She was disappointed.", "span1_index": 0, "span2_index": 11, "span1_text": "Jane", "span2_text": "She", "idx": 428, "label": 1} 23 | {"text": "The donkey wished a wart on its hind leg would disappear, and it did.", "span1_index": 8, "span2_index": 12, "span1_text": "leg", "span2_text": "it", "idx": 346, "label": 0} 24 | {"text": "Archaeologists have concluded that humans lived in Laputa 20,000 years ago. They hunted for deer on the river banks.", "span1_index": 0, "span2_index": 11, "span1_text": "Archaeologists", "span2_text": "They", "idx": 124, "label": 0} 25 | {"text": "James asked Robert for a favor but he refused.", "span1_index": 0, "span2_index": 7, "span1_text": "James", "span2_text": "he", "idx": 91, "label": 0} 26 | {"text": "There is a pillar between me and the stage , and I can't see it .", "span1_index": 7, "span2_index": 14, "span1_text": "the stage", "span2_text": "it", "idx": 450, "label": 1} 27 | {"text": "Tom said \"Check\" to Ralph as he moved his bishop.", "span1_index": 4, "span2_index": 8, "span1_text": "Ralph", "span2_text": "his", "idx": 308, "label": 0} 28 | {"text": "Do you suppose that Peter is responsible for the captain \u2019s illness? Maybe he bribed the cook to put something in his food.", "span1_index": 8, "span2_index": 13, "span1_text": "the captain", "span2_text": "he", "idx": 470, "label": 0} 29 | {"text": "Sid explained his theory to Mark but he couldn't understand him.", "span1_index": 0, "span2_index": 7, "span1_text": "Sid", "span2_text": "he", "idx": 74, "label": 0} 30 | {"text": "Susan knew that Ann 's son had been in a car accident, so she told her about it.", "span1_index": 0, "span2_index": 13, "span1_text": "Susan", "span2_text": "she", "idx": 321, "label": 1} 31 | {"text": "Do you suppose that Peter is responsible for the captain \u2019s illness? Maybe he bribed the cook to put something in his food.", "span1_index": 4, "span2_index": 13, "span1_text": "Peter", "span2_text": "he", "idx": 192, "label": 1} 32 | {"text": "Jackson was greatly influenced by Arnold , though he lived two centuries later.", "span1_index": 5, "span2_index": 8, "span1_text": "Arnold", "span2_text": "he", "idx": 367, "label": 0} 33 | -------------------------------------------------------------------------------- /pretrained_checkpoints/t011b_ia3_finish.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/r-three/t-few/114deced63ae722a368e06cb08ea956b20c393a6/pretrained_checkpoints/t011b_ia3_finish.pt -------------------------------------------------------------------------------- /pretrained_checkpoints/t03b_ia3_finish.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/r-three/t-few/114deced63ae722a368e06cb08ea956b20c393a6/pretrained_checkpoints/t03b_ia3_finish.pt -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.10.1+cu113 2 | datasets==2.0.0 3 | transformers==4.15.0 4 | pytorch-lightning==1.5.8 5 | torchmetrics==0.6.2 6 | psutil==5.9.0 7 | deepspeed==0.5.10 8 | sentencepiece==0.1.96 9 | promptsource 10 | scipy -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/r-three/t-few/114deced63ae722a368e06cb08ea956b20c393a6/src/__init__.py -------------------------------------------------------------------------------- /src/data/__init__.py: -------------------------------------------------------------------------------- 1 | from .data_module import FinetuneDataModule, PretrainDataModule, create_collate_fn, FinetuneDatasetWithTemplate 2 | from .dataset_readers import get_dataset_reader 3 | -------------------------------------------------------------------------------- /src/data/datasets.csv: -------------------------------------------------------------------------------- 1 | HF_name,subset,task_by_convention,do_train,do_eval,train_size 2 | crows_pairs,,bias_and_fairness,,BIAS_FAIRNESS, 3 | jigsaw_toxicity_pred,,bias_and_fairness,,BIAS_FAIRNESS, 4 | super_glue,axg,bias_and_fairness,,BIAS_FAIRNESS, 5 | wino_bias,type1_anti,bias_and_fairness,,BIAS_FAIRNESS, 6 | wino_bias,type2_anti,bias_and_fairness,,BIAS_FAIRNESS, 7 | wino_bias,type1_pro,bias_and_fairness,,BIAS_FAIRNESS, 8 | wino_bias,type2_pro,bias_and_fairness,,BIAS_FAIRNESS, 9 | super_glue,wsc.fixed,coreference,SGLUE,BASE,554 10 | winogrande,winogrande_xl,coreference,,BASE,40398 11 | super_glue,cb,NLI,,BASE,250 12 | super_glue,rte,NLI,,BASE,2490 13 | anli,,NLI,,BASE,162865 14 | glue,mrpc,paraphrase,BASE,,3668 15 | glue,qqp,paraphrase,BASE,,363846 16 | paws,labeled_final,paraphrase,BASE,,49401 17 | ai2_arc,ARC-Challenge,QA_closed_book,GPT_EVAL,,1119 18 | ai2_arc,ARC-Easy,QA_closed_book,GPT_EVAL,,2251 19 | kilt_tasks,hotpotqa,QA_closed_book,BASE,,88869 20 | trivia_qa,unfiltered,QA_closed_book,GPT_EVAL,,87622 21 | web_questions,,QA_closed_book,GPT_EVAL,,3778 22 | wiki_qa,,QA_closed_book,BASE,,20360 23 | adversarial_qa,dbidaf,QA_extractive,BASE,,10000 24 | adversarial_qa,dbert,QA_extractive,BASE,,10000 25 | adversarial_qa,droberta,QA_extractive,BASE,,10000 26 | duorc,SelfRC,QA_extractive,BASE,,60721 27 | duorc,ParaphraseRC,QA_extractive,BASE,,69524 28 | ropes,,QA_extractive,BASE,,10924 29 | squad_v2,,QA_extractive,GPT_EVAL,,130319 30 | super_glue,record,QA_extractive,SGLUE,,100730 31 | quoref,,QA_extractive,BASE,,19399 32 | cos_e,v1.11,QA_multiple_choice,BASE,,9741 33 | cosmos_qa,,QA_multiple_choice,BASE,,25262 34 | dream,,QA_multiple_choice,BASE,,6116 35 | openbookqa,main,QA_multiple_choice,GPT_EVAL,,4957 36 | qasc,,QA_multiple_choice,BASE,,8134 37 | quail,,QA_multiple_choice,BASE,,10246 38 | quarel,,QA_multiple_choice,BASE,,1941 39 | quartz,,QA_multiple_choice,BASE,,2696 40 | race,high,QA_multiple_choice,GPT_EVAL,,62445 41 | race,middle,QA_multiple_choice,GPT_EVAL,,25421 42 | sciq,,QA_multiple_choice,BASE,,11679 43 | social_i_qa,,QA_multiple_choice,BASE,,33410 44 | super_glue,boolq,QA_multiple_choice,SGLUE,,9427 45 | super_glue,copa,QA_multiple_choice,SGLUE,BASE,400 46 | super_glue,multirc,QA_multiple_choice,SGLUE,,27243 47 | wiki_hop,original,QA_multiple_choice,BASE,,43738 48 | wiqa,,QA_multiple_choice,BASE,,29808 49 | piqa,,QA_multiple_choice,GPT_EVAL,,16113 50 | amazon_polarity,,sentiment,BASE,,3600000 51 | app_reviews,,sentiment,BASE,,288065 52 | imdb,,sentiment,BASE,,25000 53 | rotten_tomatoes,,sentiment,BASE,,8530 54 | yelp_review_full,,sentiment,BASE,,650000 55 | story_cloze,2016,story_completion,,BASE, 56 | hellaswag,,story_completion,GPT_EVAL,BASE,39905 57 | common_gen,,structure_to_text,BASE,,67389 58 | wiki_bio,,structure_to_text,BASE,,582659 59 | cnn_dailymail,3.0.0,summarization,BASE,,287113 60 | gigaword,,summarization,BASE,,3803957 61 | multi_news,,summarization,BASE,,44972 62 | samsum,,summarization,BASE,,14732 63 | xsum,,summarization,BASE,,204045 64 | ag_news,,topic_classification,BASE,,120000 65 | dbpedia_14,,topic_classification,BASE,,560000 66 | trec,,topic_classification,BASE,,5452 67 | super_glue,wic,word_sense_disambiguation,SGLUE,BASE,5428 68 | -------------------------------------------------------------------------------- /src/data/raft_prompt_construction_settings.jsonl: -------------------------------------------------------------------------------- 1 | {"ade_corpus_v2": ["Sentence"], "banking_77": ["Query"], "terms_of_service": ["Sentence"], "tai_safety_research": ["Title", "Abstract Note", "Publication Title", "Item Type", "Publication Year"], "neurips_impact_statement_risks": ["Impact statement", "Paper title"], "overruling": ["Sentence"], "systematic_review_inclusion": ["Title", "Abstract", "Journal"], "one_stop_english": ["Article"], "tweet_eval_hate": ["Tweet"], "twitter_complaints": ["Tweet text"], "semiconductor_org_types": ["Organization name", "Paper title"]} 2 | {"ade_corpus_v2": "Label the sentence based on whether it is related to an adverse drug effect (ADE). Details are described below:\nDrugs: Names of drugs and chemicals that include brand names, trivial names, abbreviations and systematic names were annotated. Mentions of drugs or chemicals should strictly be in a therapeutic context. This category does not include the names of metabolites, reaction byproducts, or hospital chemicals (e.g. surgical equipment disinfectants).\nAdverse effect: Mentions of adverse effects include signs, symptoms, diseases, disorders, acquired abnormalities, deficiencies, organ damage or death that strictly occur as a consequence of drug intake.", "banking_77": "The following is a banking customer service query. Classify the query into one of the 77 categories available.", "terms_of_service": "Label the sentence from a Terms of Service based on whether it is potentially unfair. If it seems clearly unfair, mark it as potentially unfair.\nAccording to art. 3 of the Directive 93/13 on Unfair Terms in Consumer Contracts, a contractual term is unfair if: 1) it has not been individually negotiated; and 2) contrary to the requirement of good faith, it causes a significant imbalance in the parties rights and obligations, to the detriment of the consumer. \nDetails on types of potentially unfair clauses are found below:\nThe jurisdiction clause stipulates what courts will have the competence to adjudicate disputes under the contract. Jurisdiction clauses giving consumers a right to bring disputes in their place of residence were marked as clearly fair, whereas clauses stating that any judicial proceeding takes a residence away were marked as clearly unfair.\nThe choice of law clause specifies what law will govern the contract, meaning also what law will be applied in potential adjudication of a dispute arising under the contract. Clauses defining the applicable law as the law of the consumer's country of residence were marked as clearly fair. In every other case, the choice of law clause was considered as potentially unfair.\nThe limitation of liability clause stipulates that the duty to pay damages is limited or excluded, for certain kind of losses, under certain conditions. Clauses that explicitly affirm non-excludable providers' liabilities were marked as clearly fair. Clauses that reduce, limit, or exclude the liability of the service provider were marked as potentially unfair when concerning broad categories of losses or causes of them.\nThe unilateral change clause specifies the conditions under which the service provider could amend and modify the terms of service and/or the service itself. Such clause was always considered as potentially unfair.\nThe unilateral termination clause gives provider the right to suspend and/or terminate the service and/or the contract, and sometimes details the circumstances under which the provider claims to have a right to do so.\nThe contract by using clause stipulates that the consumer is bound by the terms of use of a specific service, simply by using the service, without even being required to mark that he or she has read and accepted them. We always marked such clauses as potentially unfair.\nThe content removal gives the provider a right to modify/delete user's content, including in-app purchases, and sometimes specifies the conditions under which the service provider may do so.\nThe arbitration clause requires or allows the parties to resolve their disputes through an arbitration process, before the case could go to court. Clauses stipulating that the arbitration should take place in a state other then the state of consumer's residence or be based on arbiter's discretion were marked as clearly unfair. Clauses defining arbitration as fully optional were marked as clearly fair.", "tai_safety_research": "Transformative AI (TAI) is defined as AI that precipitates a transition comparable to (or more significant than) the agricultural or industrial revolution. Label a paper as \"TAI safety research\" if: \n1. The contents of the paper are directly motivated by, and substantively inform, the challenge of ensuring good outcomes for TAI, \n2. There is substantive content on AI safety, not just AI capabilities, \n3. The intended audience is the community of researchers, \n4. It meets a subjective threshold of seriousness/quality, \n5. Peer review is not required.", "neurips_impact_statement_risks": "Label the impact statement based on whether it mentions a harmful application of the research done in the paper. Make sure the statement is sufficient to conclude there are harmful applications of the research being done, not a past risk that this research is solving.", "overruling": "In law, an overruling sentence is a statement that nullifies a previous case decision as a precedent, by a constitutionally valid statute or a decision by the same or higher ranking court which establishes a different rule on the point of law involved. Label the sentence based on whether it is overruling or not.", "systematic_review_inclusion": "Identify whether this paper should be included in a meta-review which includes the findings of systematic reviews on interventions designed to promote charitable donations. \nIncluded reviews should describe monetary charitable donations, assess any population of participants in any context, and be peer reviewed and written in English. \nThey should not report new data, be non-systematic reviews, consider cause-related marketing or other kinds of prosocial behaviour.", "one_stop_english": "The following is an article sourced from The Guardian newspaper, and rewritten by teachers to suit three levels of adult English as Second Language (ESL) learners: elementary, intermediate, and advanced. Predict the level of the article.", "tweet_eval_hate": "Label whether the following tweet contains hate speech against either immigrants or women. Hate Speech (HS) is commonly defined as any communication that disparages a person or a group on the basis of some characteristic such as race, color, ethnicity, gender, sexual orientation, nationality, religion, or other characteristics.", "twitter_complaints": "A complaint presents a state of affairs which breaches the writer\u2019s favorable expectation. Label the tweet text based on whether it contains a complaint.", "semiconductor_org_types": "The dataset is a list of institutions that have contributed papers to semiconductor conferences in the last 25 years, as catalogued by IEEE and sampled randomly. The goal is to classify the institutions into one of three categories: \"university\", \"company\" or \"research institute\"."} 3 | -------------------------------------------------------------------------------- /src/intrinsic_said_setup.py: -------------------------------------------------------------------------------- 1 | """Install Intrinsic SAID.""" 2 | import os 3 | import setuptools 4 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension 5 | 6 | os.environ["TORCH_CUDA_ARCH_LIST"] = "3.5;3.7;6.1;7.0;7.5;8.6+PTX" 7 | 8 | 9 | def setup_package(): 10 | long_description = "nicl" 11 | setuptools.setup( 12 | ext_modules=[ 13 | CUDAExtension( 14 | "src.models.fwh_cuda", 15 | sources=[ 16 | "src/models/fwh_cuda/fwh_cpp.cpp", 17 | "src/models/fwh_cuda/fwh_cu.cu", 18 | ], 19 | ) 20 | ], 21 | cmdclass={"build_ext": BuildExtension}, 22 | ) 23 | 24 | 25 | if __name__ == "__main__": 26 | setup_package() 27 | -------------------------------------------------------------------------------- /src/models/AdapterVariants/Adapters.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | from .VariantLayers import LowRankLinear, PHMLinear 3 | from transformers.activations import ACT2FN 4 | 5 | 6 | # From https://github.com/rabeehk/compacter 7 | 8 | ACT2FN["identity"] = lambda x: x 9 | 10 | 11 | class Adapter(nn.Module): 12 | def __init__(self, config, transformer_config): 13 | super().__init__() 14 | self.adapter_input_size = transformer_config.hidden_size 15 | self.adapter_latent_size = self.adapter_input_size // config.adapter_reduction_factor 16 | self.non_linearity = ACT2FN[config.adapter_non_linearity] 17 | self.residual = config.normal_adapter_residual 18 | 19 | # down projection 20 | self.down_proj = nn.Linear(self.adapter_input_size, self.adapter_latent_size) 21 | # up projection 22 | self.up_proj = nn.Linear(self.adapter_latent_size, self.adapter_input_size) 23 | 24 | self.init_weights() 25 | 26 | def init_weights(self): 27 | """Initialize the weights -> so that initially we the whole Adapter layer is a near-identity function""" 28 | self.down_proj.weight.data.normal_(mean=0.0, std=0.02) 29 | self.down_proj.bias.data.zero_() 30 | self.up_proj.weight.data.normal_(mean=0.0, std=0.02) 31 | self.up_proj.bias.data.zero_() 32 | 33 | def forward(self, x): 34 | output = self.up_proj(self.non_linearity(self.down_proj(x))) 35 | if self.residual: 36 | output = x + output 37 | return output 38 | 39 | 40 | class LowRankAdapter(nn.Module): 41 | """This is the low-rank adapter, in which each adapter is composed of two rank-one matrices.""" 42 | 43 | def __init__(self, config, transformer_config): 44 | super().__init__() 45 | self.config = config 46 | self.input_dim = transformer_config.hidden_size 47 | self.down_sample_size = self.input_dim // config.adapter_reduction_factor 48 | self.activation = ACT2FN[config.adapter_non_linearity] 49 | self.down_sampler = LowRankLinear( 50 | self.input_dim, 51 | self.down_sample_size, 52 | w_init=config.lowrank_adapter_w_init, 53 | rank=config.lowrank_adapter_rank, 54 | ) 55 | self.up_sampler = LowRankLinear( 56 | self.down_sample_size, 57 | self.input_dim, 58 | w_init=config.lowrank_adapter_w_init, 59 | rank=config.lowrank_adapter_rank, 60 | ) 61 | 62 | def forward(self, x): 63 | z = self.down_sampler(x) 64 | z = self.activation(z) 65 | output = self.up_sampler(z) 66 | return output 67 | 68 | 69 | class HyperComplexAdapter(nn.Module): 70 | """Hypercomplex Adapter layer, in which the weights of up and down sampler modules 71 | are parameters are 1/n times of the conventional adapter layers, where n is 72 | hypercomplex division number.""" 73 | 74 | def __init__(self, config, transformer_config): 75 | super().__init__() 76 | self.config = config 77 | self.input_dim = transformer_config.hidden_size 78 | self.down_sample_size = self.input_dim // config.adapter_reduction_factor 79 | self.activation = ACT2FN[config.adapter_non_linearity] 80 | self.down_sampler = PHMLinear( 81 | in_features=self.input_dim, 82 | out_features=self.down_sample_size, 83 | bias=True, 84 | c_init=config.compacter_phm_c_init, 85 | phm_dim=config.compacter_hypercomplex_division, 86 | learn_phm=config.compacter_learn_phm, 87 | w_init=config.compacter_hypercomplex_nonlinearity, 88 | shared_phm_rule=config.compacter_shared_phm_rule, 89 | factorized_phm=config.compacter_factorized_phm, 90 | shared_W_phm=config.compacter_shared_W_phm, 91 | factorized_phm_rule=config.compacter_factorized_phm_rule, 92 | phm_rank=config.compacter_phm_rank, 93 | phm_init_range=config.compacter_phm_init_range, 94 | kronecker_prod=config.compacter_kronecker_prod, 95 | ) 96 | self.up_sampler = PHMLinear( 97 | in_features=self.down_sample_size, 98 | out_features=self.input_dim, 99 | bias=True, 100 | c_init=config.compacter_phm_c_init, 101 | phm_dim=config.compacter_hypercomplex_division, 102 | learn_phm=config.compacter_learn_phm, 103 | w_init=config.compacter_hypercomplex_nonlinearity, 104 | shared_phm_rule=config.compacter_shared_phm_rule, 105 | factorized_phm=config.compacter_factorized_phm, 106 | shared_W_phm=config.compacter_shared_W_phm, 107 | factorized_phm_rule=config.compacter_factorized_phm_rule, 108 | phm_rank=config.compacter_phm_rank, 109 | phm_init_range=config.compacter_phm_init_range, 110 | kronecker_prod=config.compacter_kronecker_prod, 111 | ) 112 | 113 | def forward(self, x): 114 | z = self.down_sampler(x) 115 | z = self.activation(z) 116 | z = self.up_sampler(z) 117 | return x + z 118 | -------------------------------------------------------------------------------- /src/models/bitfit.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import re 4 | 5 | 6 | def modify_with_bitfit(transformer, config): 7 | for m_name, module in dict(transformer.named_modules()).items(): 8 | if re.fullmatch(config.bitfit_modules, m_name): 9 | for c_name, layer in dict(module.named_children()).items(): 10 | if re.fullmatch(config.bitfit_layers, c_name): 11 | layer.bias = nn.Parameter(torch.zeros(layer.out_features)) 12 | return transformer 13 | 14 | 15 | if __name__ == "__main__": 16 | from transformers import AutoModelForSeq2SeqLM, AutoTokenizer 17 | 18 | class BitFitConfig: 19 | def __init__(self): 20 | self.bitfit_modules = ".*" 21 | self.bitfit_layers = "q|k|v|o|w.*" 22 | self.trainable_param_names = ".*layer_norm.*|.*bias" 23 | # lora_modules and lora_layers are speicified with regular expressions 24 | # see https://www.w3schools.com/python/python_regex.asp for reference 25 | 26 | config = BitFitConfig() 27 | model = AutoModelForSeq2SeqLM.from_pretrained("t5-small") 28 | tokenizer = AutoTokenizer.from_pretrained("t5-small") 29 | 30 | input_seq = tokenizer( 31 | ["Applies a linear transformation to the incoming data."], 32 | return_tensors="pt", 33 | ) 34 | target_seq = tokenizer( 35 | ["Parameters: in_features - size of each input sample. out_features - size of each output sample."], 36 | return_tensors="pt", 37 | ) 38 | 39 | print("Old model") 40 | print(model) 41 | with torch.no_grad(): 42 | old_outputs = model( 43 | input_ids=input_seq.input_ids, 44 | decoder_input_ids=target_seq.input_ids[:, :-1], 45 | labels=target_seq.input_ids[:, 1:], 46 | ) 47 | 48 | model = modify_with_bitfit(model, config) 49 | 50 | print("New model") 51 | print(model) 52 | with torch.no_grad(): 53 | new_outputs = model( 54 | input_ids=input_seq.input_ids, 55 | decoder_input_ids=target_seq.input_ids[:, :-1], 56 | labels=target_seq.input_ids[:, 1:], 57 | ) 58 | 59 | print("Trainable parameters") 60 | print( 61 | [ 62 | p_name 63 | for p_name in dict(model.named_parameters()).keys() 64 | if re.fullmatch(config.trainable_param_names, p_name) 65 | ] 66 | ) 67 | 68 | print(f"Logits diff {torch.abs(old_outputs.logits - new_outputs.logits).mean():.3f}") 69 | print(f"Loss diff old={old_outputs.loss:.3f} new={new_outputs.loss:.3f}") 70 | -------------------------------------------------------------------------------- /src/models/fishmask.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | 4 | 5 | def fishmask_plugin_on_init(pl_module): 6 | if pl_module.config.fishmask_mode == "apply": 7 | print(f"Load gradient mask from {pl_module.config.fishmask_path}") 8 | mask_dict = torch.load(pl_module.config.fishmask_path) 9 | for param_name, param in pl_module.model.named_parameters(): 10 | param.stored_mask = mask_dict[param_name].to("cuda") 11 | 12 | 13 | def fishmask_plugin_on_optimizer_step(pl_module): 14 | if pl_module.config.fishmask_mode == "create": 15 | for name, param in pl_module.model.named_parameters(): 16 | if not hasattr(param, "stored_grad"): 17 | param.stored_grad = torch.zeros_like(param.data) 18 | param.stored_grad += torch.square(param.grad) / pl_module.config.num_shot 19 | param.grad.zero_() 20 | elif pl_module.config.fishmask_mode == "apply": 21 | for name, param in pl_module.model.named_parameters(): 22 | param.grad.data *= param.stored_mask 23 | else: 24 | raise ValueError(f"Invalid fishmask_mode {pl_module.config.fishmask_mode}") 25 | 26 | 27 | def fishmask_plugin_on_end(pl_module): 28 | if pl_module.config.fishmask_mode == "create": 29 | sizes = {} 30 | tensors = [] 31 | all_params_size = 0 32 | for param_name, param in pl_module.model.named_parameters(): 33 | sizes[param_name] = param.size() 34 | tensors.append(param.stored_grad.reshape(-1).to("cpu")) 35 | all_params_size += param.numel() 36 | tensors = torch.cat(tensors, 0) 37 | keep_num = int(all_params_size * pl_module.config.fishmask_keep_ratio) 38 | assert keep_num > 0 39 | top_pos = torch.topk(tensors, keep_num)[1] 40 | masks = torch.zeros(tensors.shape, dtype=torch.bool) 41 | masks[top_pos] = True 42 | del tensors 43 | assert masks.long().sum() == len(top_pos) 44 | mask_dict = {} 45 | 46 | now_idx = 0 47 | for param_name, param_size in sizes.items(): 48 | end_idx = now_idx + param_size.numel() 49 | mask_dict[param_name] = masks[now_idx:end_idx].reshape(param_size) 50 | now_idx = end_idx 51 | assert now_idx == len(masks) 52 | 53 | all_params_size = 0 54 | trainable_weight_size = 0 55 | for param_name, param_mask in mask_dict.items(): 56 | trainable_weight_size += param_mask.long().sum().item() 57 | all_params_size += param_mask.numel() 58 | 59 | print(f"Trainable parameters: {(trainable_weight_size) / all_params_size * 100:.3f} %") 60 | fishmask_path = os.path.join(pl_module.config.exp_dir, "mask.bin") 61 | torch.save(mask_dict, fishmask_path) 62 | print(f"Save gradient mask to {fishmask_path}") 63 | -------------------------------------------------------------------------------- /src/models/fwh_cuda/fwh_cpp.cpp: -------------------------------------------------------------------------------- 1 | // The codes are from Armen Aghajanyan from facebook, from paper 2 | // Intrinsic Dimensionality Explains the Effectiveness of Language Model Fine-Tuning 3 | // https://arxiv.org/abs/2012.13255 4 | 5 | 6 | #include 7 | 8 | #include 9 | 10 | // CUDA forward declarations 11 | 12 | void fast_walsh_hadamard_transform_cuda_kernel(const int NN, const int halfLL, torch::Tensor in, torch::Tensor out, bool normalize); 13 | 14 | // C++ interface 15 | 16 | #define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") 17 | #define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") 18 | #define CHECK_INPUT(x) \ 19 | CHECK_CUDA(x); \ 20 | CHECK_CONTIGUOUS(x) 21 | 22 | torch::Tensor fast_walsh_hadamard_transform(torch::Tensor input, bool normalize) 23 | { 24 | CHECK_INPUT(input); 25 | const int NN = input.numel(); 26 | torch::Tensor output_flat = input.clone(); 27 | int ll = 0; 28 | int LL = 1; 29 | while (LL < NN) 30 | { 31 | ll += 1; 32 | LL *= 2; 33 | } 34 | const int halfLL = LL / 2; 35 | fast_walsh_hadamard_transform_cuda_kernel(NN, halfLL, input, output_flat, normalize); 36 | return output_flat; 37 | } 38 | 39 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) 40 | { 41 | m.def("fast_walsh_hadamard_transform", &fast_walsh_hadamard_transform, "Fast Walsh Hadamard Transform (CUDA)"); 42 | } 43 | -------------------------------------------------------------------------------- /src/models/fwh_cuda/fwh_cu.cu: -------------------------------------------------------------------------------- 1 | // The codes are from Armen Aghajanyan from facebook, from paper 2 | // Intrinsic Dimensionality Explains the Effectiveness of Language Model Fine-Tuning 3 | // https://arxiv.org/abs/2012.13255 4 | 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | template 13 | __global__ void FastWalshHadamardKernel(const int stride, const scalar_t* in, scalar_t* out) { 14 | const auto idx = (threadIdx.x + blockIdx.x * blockDim.x); 15 | const auto elemIdx = (idx / stride ) * (2 * stride) + (idx % stride); 16 | const auto tmp = in[elemIdx], tmp2 = in[elemIdx + stride]; 17 | out[elemIdx] = tmp + tmp2; 18 | out[elemIdx + stride] = tmp - tmp2; 19 | } 20 | 21 | template 22 | __global__ void FastWalshHadamardSubKernel(const scalar_t scalar, scalar_t* out) { 23 | const auto idx = (threadIdx.x + blockIdx.x * blockDim.x); 24 | out[idx] *= scalar; 25 | } 26 | 27 | 28 | void fast_walsh_hadamard_transform_cuda_kernel(const int NN, const int halfLL, torch::Tensor in, torch::Tensor out, bool normalize) { 29 | // Apply Unnormalized Fast Walsh Hadamard transform 30 | int stride = halfLL; 31 | float normalizer = 1.0; 32 | float sqrt2inv = 0.70710678118654746; 33 | 34 | while (stride >= 1) { 35 | if(stride == halfLL) 36 | { 37 | AT_DISPATCH_FLOATING_TYPES(in.scalar_type(),"fast_walsh_hadamard_transform_in", ([&] { 38 | FastWalshHadamardKernel<<>>(stride, in.data_ptr(), out.data_ptr()); 39 | })); 40 | } 41 | else 42 | { 43 | AT_DISPATCH_FLOATING_TYPES(in.scalar_type(),"fast_walsh_hadamard_transform_out", ([&] { 44 | FastWalshHadamardKernel<<>>(stride, out.data_ptr(), out.data_ptr()); 45 | })); 46 | } 47 | 48 | stride /= 2; 49 | normalizer *= sqrt2inv; 50 | } 51 | if(normalize){ 52 | AT_DISPATCH_FLOATING_TYPES(in.scalar_type(),"fast_walsh_hadamard_transform_final", ([&] { 53 | FastWalshHadamardSubKernel<<>>(normalizer, out.data_ptr()); 54 | })); 55 | } 56 | } -------------------------------------------------------------------------------- /src/models/lora.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import re 5 | 6 | 7 | class LoRALinear(nn.Module): 8 | def __init__(self, linear_layer, rank, scaling_rank, init_scale): 9 | super().__init__() 10 | self.in_features = linear_layer.in_features 11 | self.out_features = linear_layer.out_features 12 | self.rank = rank 13 | self.scaling_rank = scaling_rank 14 | self.weight = linear_layer.weight 15 | self.bias = linear_layer.bias 16 | if self.rank > 0: 17 | self.lora_a = nn.Parameter(torch.randn(rank, linear_layer.in_features) * init_scale) 18 | if init_scale < 0: 19 | self.lora_b = nn.Parameter(torch.randn(linear_layer.out_features, rank) * init_scale) 20 | else: 21 | self.lora_b = nn.Parameter(torch.zeros(linear_layer.out_features, rank)) 22 | if self.scaling_rank: 23 | self.multi_lora_a = nn.Parameter( 24 | torch.ones(self.scaling_rank, linear_layer.in_features) 25 | + torch.randn(self.scaling_rank, linear_layer.in_features) * init_scale 26 | ) 27 | if init_scale < 0: 28 | self.multi_lora_b = nn.Parameter( 29 | torch.ones(linear_layer.out_features, self.scaling_rank) 30 | + torch.randn(linear_layer.out_features, self.scaling_rank) * init_scale 31 | ) 32 | else: 33 | self.multi_lora_b = nn.Parameter(torch.ones(linear_layer.out_features, self.scaling_rank)) 34 | 35 | def forward(self, input): 36 | if self.scaling_rank == 1 and self.rank == 0: 37 | # parsimonious implementation for ia3 and lora scaling 38 | if self.multi_lora_a.requires_grad: 39 | hidden = F.linear((input * self.multi_lora_a.flatten()), self.weight, self.bias) 40 | else: 41 | hidden = F.linear(input, self.weight, self.bias) 42 | if self.multi_lora_b.requires_grad: 43 | hidden = hidden * self.multi_lora_b.flatten() 44 | return hidden 45 | else: 46 | # general implementation for lora (adding and scaling) 47 | weight = self.weight 48 | if self.scaling_rank: 49 | weight = weight * torch.matmul(self.multi_lora_b, self.multi_lora_a) / self.scaling_rank 50 | if self.rank: 51 | weight = weight + torch.matmul(self.lora_b, self.lora_a) / self.rank 52 | return F.linear(input, weight, self.bias) 53 | 54 | def extra_repr(self): 55 | return "in_features={}, out_features={}, bias={}, rank={}, scaling_rank={}".format( 56 | self.in_features, self.out_features, self.bias is not None, self.rank, self.scaling_rank 57 | ) 58 | 59 | 60 | def modify_with_lora(transformer, config): 61 | for m_name, module in dict(transformer.named_modules()).items(): 62 | if re.fullmatch(config.lora_modules, m_name): 63 | for c_name, layer in dict(module.named_children()).items(): 64 | if re.fullmatch(config.lora_layers, c_name): 65 | assert isinstance( 66 | layer, nn.Linear 67 | ), f"LoRA can only be applied to torch.nn.Linear, but {layer} is {type(layer)}." 68 | setattr( 69 | module, 70 | c_name, 71 | LoRALinear(layer, config.lora_rank, config.lora_scaling_rank, config.lora_init_scale), 72 | ) 73 | return transformer 74 | 75 | 76 | if __name__ == "__main__": 77 | from transformers import AutoModelForSeq2SeqLM, AutoTokenizer 78 | 79 | class LoRAConfig: 80 | def __init__(self): 81 | self.lora_rank = 4 82 | self.lora_init_scale = 0.01 83 | self.lora_modules = ".*SelfAttention|.*EncDecAttention" 84 | self.lora_layers = "q|k|v|o" 85 | self.trainable_param_names = ".*layer_norm.*|.*lora_[ab].*" 86 | self.lora_scaling_rank = 1 87 | # lora_modules and lora_layers are speicified with regular expressions 88 | # see https://www.w3schools.com/python/python_regex.asp for reference 89 | 90 | config = LoRAConfig() 91 | model = AutoModelForSeq2SeqLM.from_pretrained("t5-small") 92 | tokenizer = AutoTokenizer.from_pretrained("t5-small") 93 | 94 | input_seq = tokenizer( 95 | ["Applies a linear transformation to the incoming data."], 96 | return_tensors="pt", 97 | ) 98 | target_seq = tokenizer( 99 | ["Parameters: in_features - size of each input sample. out_features - size of each output sample."], 100 | return_tensors="pt", 101 | ) 102 | 103 | print("Old model") 104 | print(model) 105 | with torch.no_grad(): 106 | old_outputs = model( 107 | input_ids=input_seq.input_ids, 108 | decoder_input_ids=target_seq.input_ids[:, :-1], 109 | labels=target_seq.input_ids[:, 1:], 110 | ) 111 | 112 | model = modify_with_lora(model, config) 113 | 114 | print("New model") 115 | print(model) 116 | with torch.no_grad(): 117 | new_outputs = model( 118 | input_ids=input_seq.input_ids, 119 | decoder_input_ids=target_seq.input_ids[:, :-1], 120 | labels=target_seq.input_ids[:, 1:], 121 | ) 122 | 123 | print("Trainable parameters") 124 | print( 125 | [ 126 | p_name 127 | for p_name in dict(model.named_parameters()).keys() 128 | if re.fullmatch(config.trainable_param_names, p_name) 129 | ] 130 | ) 131 | 132 | print(f"Logits diff {torch.abs(old_outputs.logits - new_outputs.logits).mean():.3f}") 133 | print(f"Loss diff old={old_outputs.loss:.3f} new={new_outputs.loss:.3f}") 134 | -------------------------------------------------------------------------------- /src/models/modify_model.py: -------------------------------------------------------------------------------- 1 | from .lora import modify_with_lora 2 | from .adapters import modify_with_adapters 3 | from .bitfit import modify_with_bitfit 4 | from .prompt_tuning import modify_with_prompt_tuning 5 | from .prefix_tuning import modify_with_prefix_tuning 6 | 7 | modifier_dict = { 8 | "lora": modify_with_lora, 9 | "bitfit": modify_with_bitfit, 10 | "adapters": modify_with_adapters, 11 | "prompt-tuning": modify_with_prompt_tuning, 12 | "prefix-tuning": modify_with_prefix_tuning, 13 | } 14 | 15 | 16 | def modify_transformer(transformer, config): 17 | if config.model_modifier == "intrinsic": 18 | from .intrinsic import modify_with_intrinsic_model 19 | 20 | modifier_dict["intrinsic"] = modify_with_intrinsic_model 21 | 22 | if config.model_modifier: 23 | if config.model_modifier in modifier_dict: 24 | transformer = modifier_dict[config.model_modifier](transformer, config) 25 | else: 26 | raise ValueError(f"Model modifier '{config.model_modifier}' not found.") 27 | 28 | return transformer 29 | -------------------------------------------------------------------------------- /src/models/prompt_tuning.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | def sample_embed(embed, sample_size, start_idx, end_idx): 6 | embed_weight = embed.weight 7 | rand_idx = torch.randint(start_idx, end_idx, (sample_size,)) 8 | return embed_weight[rand_idx].detach() 9 | 10 | 11 | class T5EncoderPromptTuningWrapper(nn.Module): 12 | def __init__(self, encoder, config): 13 | super().__init__() 14 | self.num_prefix_emb = config.prompt_tuning_num_prefix_emb 15 | self.prompt_tuning_encoder = config.prompt_tuning_encoder 16 | self.encoder = encoder 17 | self.prompt_embedding = nn.Parameter( 18 | sample_embed( 19 | embed=encoder.get_input_embeddings(), 20 | sample_size=self.num_prefix_emb, 21 | start_idx=3, 22 | end_idx=5003, 23 | ) 24 | ) # [num_prefix_emb, emb_dim] sampled from 5000 most common regular token embeddings 25 | 26 | def forward(self, input_ids, attention_mask, inputs_embeds=None, **kwargs): 27 | bs = input_ids.size(0) 28 | inputs_embeds = self.encoder.embed_tokens(input_ids) # [bs, max_seq_len, d_emb] 29 | prompt_attention_mask = attention_mask.new_ones((bs, self.num_prefix_emb)) # [bs, prompt_len] 30 | if self.prompt_tuning_encoder: 31 | inputs_embeds = torch.cat( 32 | [self.prompt_embedding[None, :, :].repeat((bs, 1, 1)), inputs_embeds], dim=1 33 | ) # [bs, prompt_len+max_seq_len, d_emb] 34 | attention_mask = torch.cat([prompt_attention_mask, attention_mask], dim=1) # [bs, prompt_len+max_seq_len] 35 | 36 | encoder_outputs = self.encoder( 37 | input_ids=None, inputs_embeds=inputs_embeds, attention_mask=attention_mask, **kwargs 38 | ) 39 | return encoder_outputs 40 | 41 | 42 | class T5DecoderPromptTuningWrapper(nn.Module): 43 | def __init__(self, decoder, config): 44 | super().__init__() 45 | self.num_prefix_emb = config.prompt_tuning_num_prefix_emb 46 | self.prompt_tuning_encoder = config.prompt_tuning_encoder 47 | self.prompt_tuning_decoder = config.prompt_tuning_decoder 48 | self.decoder = decoder 49 | self.prompt_embedding = nn.Parameter( 50 | sample_embed( 51 | embed=decoder.get_input_embeddings(), 52 | sample_size=self.num_prefix_emb, 53 | start_idx=3, 54 | end_idx=5003, 55 | ) 56 | ) # [num_prefix_emb, emb_dim] sampled from 5000 most common regular token embeddings 57 | 58 | def forward(self, input_ids, attention_mask, encoder_attention_mask, inputs_embeds=None, **kwargs): 59 | bs = input_ids.size(0) 60 | inputs_embeds = self.decoder.embed_tokens(input_ids) # [bs, max_seq_len, d_emb] 61 | prompt_attention_mask = attention_mask.new_ones((bs, self.num_prefix_emb)) # [bs, prompt_len] 62 | if self.prompt_tuning_encoder: 63 | encoder_attention_mask = torch.cat( 64 | [prompt_attention_mask, encoder_attention_mask], dim=1 65 | ) # [bs, prompt_len+max_seq_len] 66 | if self.prompt_tuning_decoder: 67 | inputs_embeds = torch.cat( 68 | [self.prompt_embedding[None, :, :].repeat((bs, 1, 1)), inputs_embeds], dim=1 69 | ) # [bs, prompt_len+max_seq_len, d_emb] 70 | attention_mask = torch.cat([prompt_attention_mask, attention_mask], dim=1) # [bs, prompt_len+max_seq_len] 71 | 72 | decoder_outputs = self.decoder( 73 | input_ids=None, 74 | attention_mask=attention_mask, 75 | encoder_attention_mask=encoder_attention_mask, 76 | inputs_embeds=inputs_embeds, 77 | **kwargs 78 | ) 79 | if self.prompt_tuning_decoder: 80 | decoder_outputs.last_hidden_state = decoder_outputs.last_hidden_state[ 81 | :, self.num_prefix_emb : 82 | ] # [bs, max_seq_len, d_emb] 83 | return decoder_outputs 84 | 85 | 86 | def modify_with_prompt_tuning(transformer, config): 87 | transformer.encoder = T5EncoderPromptTuningWrapper(transformer.encoder, config) 88 | transformer.decoder = T5DecoderPromptTuningWrapper(transformer.decoder, config) 89 | return transformer 90 | -------------------------------------------------------------------------------- /src/pl_train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import argparse 4 | from datetime import datetime 5 | from transformers import AutoTokenizer, AutoModelForSeq2SeqLM 6 | from pytorch_lightning import Trainer 7 | from pytorch_lightning.loggers import TensorBoardLogger 8 | 9 | from src.data import FinetuneDataModule, get_dataset_reader, PretrainDataModule 10 | from src.models.EncoderDecoder import EncoderDecoder 11 | from src.models.modify_model import modify_transformer 12 | from src.utils.Config import Config 13 | from src.utils.util import ParseKwargs, set_seeds 14 | 15 | 16 | def get_transformer(config): 17 | tokenizer = AutoTokenizer.from_pretrained(config.origin_model) 18 | model = AutoModelForSeq2SeqLM.from_pretrained(config.origin_model, low_cpu_mem_usage=True) 19 | 20 | tokenizer.model_max_length = config.max_seq_len 21 | model = modify_transformer(model, config) 22 | return tokenizer, model 23 | 24 | 25 | def main(config): 26 | """ 27 | Trains the model 28 | 29 | :param config: 30 | :return: 31 | """ 32 | 33 | tokenizer, model = get_transformer(config) 34 | dataset_reader = get_dataset_reader(config) 35 | if config.dataset == "T0Mixture": 36 | datamodule = PretrainDataModule(config, tokenizer, dataset_reader) 37 | else: 38 | datamodule = FinetuneDataModule(config, tokenizer, dataset_reader) 39 | model = EncoderDecoder(config, tokenizer, model, dataset_reader) 40 | logger = TensorBoardLogger(config.exp_dir, name="log") 41 | 42 | trainer = Trainer( 43 | enable_checkpointing=False, 44 | gpus=torch.cuda.device_count(), 45 | precision=config.compute_precision, 46 | amp_backend="native", 47 | strategy=config.compute_strategy if config.compute_strategy != "none" else None, 48 | logger=logger, 49 | log_every_n_steps=4, 50 | max_steps=config.num_steps, 51 | min_steps=config.num_steps, 52 | num_sanity_val_steps=-1 if config.eval_before_training else 0, 53 | check_val_every_n_epoch=config.eval_epoch_interval, 54 | accumulate_grad_batches=config.grad_accum_factor, 55 | gradient_clip_val=config.grad_clip_norm, 56 | ) 57 | trainer.fit(model, datamodule) 58 | 59 | 60 | if __name__ == "__main__": 61 | parser = argparse.ArgumentParser() 62 | parser.add_argument("-c", "--config_files", required=True) 63 | parser.add_argument("-k", "--kwargs", nargs="*", action=ParseKwargs, default={}) 64 | args = parser.parse_args() 65 | 66 | config = Config(args.config_files, args.kwargs) 67 | print(f"Start experiment {config.exp_name}") 68 | # Setup config 69 | assert config.compute_strategy in ["none", "ddp", "deepspeed_stage_3_offload", "deepspeed_stage_3"] 70 | if config.fishmask_mode == "create": 71 | print("Detecting fishmask_mode=create, override batch_size, num_step, fishmask_path") 72 | config.batch_size = 1 73 | config.num_steps = config.num_shot 74 | config.eval_before_training = False 75 | config.fishmask_path = None 76 | 77 | print(config.to_json()) 78 | 79 | if config.allow_skip_exp and os.path.exists(config.finish_flag_file): 80 | print(f"Skip finished experiment {config.exp_name}") 81 | else: 82 | print(f"Mark experiment {config.exp_name} as claimed") 83 | with open(config.finish_flag_file, "a+") as f: 84 | f.write(datetime.now().strftime("%m/%d/%Y %H:%M:%S") + "\n") 85 | set_seeds(config.seed) 86 | main(config) 87 | -------------------------------------------------------------------------------- /src/scripts/eval_template.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import argparse 3 | 4 | dict_dataset_2_template_idx = { 5 | "copa": list(range(12)), 6 | "h-swag": [3, 4, 8, 10], 7 | "storycloze": [0, 1, 3, 4, 5], 8 | "winogrande": [0, 2, 3, 4, 5], 9 | "wsc": list(range(10)), 10 | "wic": list(range(10)), 11 | "rte": list(range(10)), 12 | "cb": list(range(15)), 13 | "anli-r1": list(range(15)), 14 | "anli-r2": list(range(15)), 15 | "anli-r3": list(range(15)), 16 | } 17 | 18 | 19 | def eval_random_template(model, method, descriptor): 20 | 21 | for seed in [0, 1, 32, 42, 1024]: 22 | for dataset in ["copa", "h-swag", "storycloze", "winogrande", "wic", "wsc", "rte", "cb", "anli-r1", "anli-r2", "anli-r3"]: 23 | if descriptor is None: 24 | command = f"bash bin/eval-template.sh {seed} {model} {method} {dataset} -1" 25 | else: 26 | command = f"bash bin/eval-template-with-descriptor.sh {seed} {model} {method} {dataset} -1 {descriptor}" 27 | subprocess.run([command], stdout=subprocess.PIPE, shell=True) 28 | 29 | 30 | def eval_all_templates(model, method, descriptor): 31 | 32 | for seed in [0, 1, 32, 42, 1024]: 33 | for dataset in ["copa", "h-swag", "storycloze", "winogrande", "wic", "wsc", "rte", "cb", "anli-r1", "anli-r2", "anli-r3"]: 34 | for template_idx in dict_dataset_2_template_idx[dataset]: 35 | if descriptor is None: 36 | command = f"bash bin/eval-template.sh {seed} {model} {method} {dataset} {template_idx}" 37 | else: 38 | command = f"bash bin/eval-template-with-descriptor.sh {seed} {model} {method} {dataset} {template_idx} {descriptor}" 39 | subprocess.run([command], stdout=subprocess.PIPE, shell=True) 40 | 41 | 42 | if __name__ == "__main__": 43 | parser = argparse.ArgumentParser() 44 | parser.add_argument("-a", "--all_template_or_random_template", required=True, choices=["all", "random"]) 45 | parser.add_argument("-model", "--model", required=True) 46 | parser.add_argument("-method", "--method", required=True) 47 | parser.add_argument("-d", "--descriptor", default=None) 48 | args = parser.parse_args() 49 | 50 | 51 | if args.all_template_or_random_template == "all": 52 | eval_all_templates(args.model, args.method, args.descriptor) 53 | else: 54 | eval_random_template(args.model, args.method, args.descriptor) 55 | 56 | -------------------------------------------------------------------------------- /src/scripts/few_glue.py: -------------------------------------------------------------------------------- 1 | 2 | import seqio 3 | import os 4 | import json 5 | import functools 6 | 7 | import tensorflow as tf 8 | 9 | from prompt_tuning.data import features 10 | from prompt_tuning.data import metrics as pt_metrics 11 | from prompt_tuning.data import postprocessors as pt_postprocessors 12 | from prompt_tuning.data import preprocessors as pt_preprocessors 13 | from prompt_tuning.data import utils 14 | import seqio 15 | from t5.data.glue_utils import get_glue_metric 16 | from t5.data.glue_utils import get_glue_postprocess_fn 17 | from t5.data.glue_utils import get_glue_text_preprocessor 18 | from t5.data.glue_utils import get_glue_weight_mapping 19 | import tensorflow_datasets as tfds 20 | 21 | import t5.data.tasks as t5_tasks 22 | 23 | # Aliases for ease of use when you want features for a specific model. 24 | T5_FEATURES = t5_tasks.DEFAULT_OUTPUT_FEATURES 25 | 26 | glue_indexer = utils.task_mapping( 27 | tuple(b.name for b in tfds.text.glue.Glue.builder_configs.values()), 28 | { 29 | "ax": "mnli", 30 | "mnli_mismatched": "mnli", 31 | "mnli_matched": "mnli" 32 | }, 33 | ) 34 | 35 | 36 | DATASET_KEY_MAPPINGS = { 37 | "rte": {"premise": "sentence1", 38 | "hypothesis": "sentence2", 39 | "idx": "idx", 40 | "label": "label"} 41 | } 42 | 43 | 44 | DATASET_OUTPUT_SIGNATURES = { 45 | "rte": {"sentence1": tf.TensorSpec(shape=(), dtype=tf.string), 46 | "sentence2": tf.TensorSpec(shape=(), dtype=tf.string), 47 | "idx": tf.TensorSpec(shape=(), dtype=tf.int32), 48 | "label": tf.TensorSpec(shape=(), dtype=tf.int64)} 49 | } 50 | 51 | 52 | def read_dataset(dataset_name, num_shot, few_shot_seed): 53 | ''' 54 | Return dataset as list of json dictionaries 55 | 56 | :param dataset_name: name of dataset 57 | :param num_shot: number of examples per class 58 | :param few_shot_seed: seed used to sample few-shot examples 59 | ''' 60 | fp = os.path.join("data", "few_shot", dataset_name, f"{num_shot}_shot", f"{few_shot_seed}_seed.jsonl") 61 | print("fp: ", fp) 62 | key_mappings = DATASET_KEY_MAPPINGS[dataset_name] 63 | 64 | with open(fp, "r") as fin: 65 | data = [] 66 | for idx, line in enumerate(fin.readlines()): 67 | original_json = json.loads(line.strip("\n")) 68 | print(original_json) 69 | updated_json = {} 70 | for (k, v) in original_json.items(): 71 | updated_json[key_mappings[k]] = v 72 | yield updated_json 73 | 74 | 75 | 76 | list_datasets = ["rte"] 77 | list_num_shot = [32] 78 | list_few_shot_seed = [32] 79 | 80 | 81 | 82 | 83 | 84 | for dataset_name in list_datasets: 85 | 86 | builder_config = tfds.text.glue.Glue.builder_configs[dataset_name] 87 | postprocess_fn = get_glue_postprocess_fn(builder_config) 88 | metric_fns = get_glue_metric(builder_config.name) 89 | 90 | validation_ds = seqio.TfdsDataSource(tfds_name=f"glue/{builder_config.name}:2.0.0").get_dataset("validation") 91 | 92 | for num_shot in list_num_shot: 93 | for few_shot_seed in list_few_shot_seed: 94 | 95 | train_ds = tf.data.Dataset.from_generator( 96 | functools.partial(read_dataset, dataset_name=dataset_name, num_shot=num_shot, few_shot_seed=few_shot_seed), output_signature=DATASET_OUTPUT_SIGNATURES[dataset_name] 97 | ) 98 | 99 | def dataset_fn(split, shuffle_files, seed): 100 | if split == "train": 101 | return train_ds 102 | elif split == "validation": 103 | return validation_ds 104 | else: 105 | raise ValueError(f"Invalid split {split}") 106 | 107 | data_source = seqio.FunctionDataSource( 108 | dataset_fn, 109 | splits=["train", "validation"], 110 | num_input_examples={"train": num_shot, "validation": len(validation_ds)}, 111 | ) 112 | # Has task index as first token 113 | seqio.TaskRegistry.add( 114 | f"glue_{dataset_name}_{num_shot}_shot_{few_shot_seed}_seed", 115 | source=data_source, 116 | preprocessors=[ 117 | get_glue_text_preprocessor(builder_config), 118 | pt_preprocessors.remove_first_text_token, 119 | seqio.preprocessors.tokenize, 120 | functools.partial( 121 | pt_preprocessors.add_sentinel_to_beginning, 122 | field="inputs", 123 | offset=glue_indexer[builder_config.name]), 124 | seqio.CacheDatasetPlaceholder(), 125 | seqio.preprocessors.append_eos_after_trim, 126 | ], 127 | postprocess_fn=postprocess_fn, 128 | metric_fns=metric_fns, 129 | output_features=T5_FEATURES) 130 | 131 | -------------------------------------------------------------------------------- /src/scripts/get_result_table.py: -------------------------------------------------------------------------------- 1 | from glob import glob 2 | import json 3 | from collections import defaultdict 4 | from scipy.stats import iqr 5 | from numpy import median 6 | import os 7 | import argparse 8 | 9 | 10 | def make_result_table(args): 11 | def collect_exp_scores(exp_name_template, datasets): 12 | print("=" * 80) 13 | all_files = glob( 14 | os.path.join(os.getenv("OUTPUT_PATH", default="exp_out"), exp_name_template, "dev_scores.json") 15 | ) 16 | print(f"Find {len(all_files)} experiments fit into {exp_name_template}") 17 | 18 | def read_last_eval(fname): 19 | with open(fname) as f: 20 | e = json.loads(f.readlines()[-1]) 21 | return e["accuracy"] 22 | 23 | acc_by_dataset = defaultdict(lambda: list()) 24 | 25 | def parse_expname(fname): 26 | elements = fname.split("/")[-2].split("_") 27 | return tuple(elements[:3] + ["_".join(elements[3:])]) 28 | 29 | for fname in all_files: 30 | result = read_last_eval(fname) 31 | model, dataset, seed, spec = parse_expname(fname) 32 | acc_by_dataset[dataset].append(result) 33 | 34 | def result_str(acc_list): 35 | if len(acc_list) > 1: 36 | return f"{median(acc_list) * 100:.2f} ({iqr(acc_list) * 100:.2f})" 37 | else: 38 | return f"{acc_list[0] * 100:.2f}" 39 | 40 | outputs = [] 41 | for dataset in datasets: 42 | acc_list = acc_by_dataset[dataset] 43 | outputs.append(result_str(acc_list)) 44 | 45 | print(", ".join([f"{dataset}: {value}" for dataset, value in zip(datasets, outputs)])) 46 | return ",".join(outputs) 47 | 48 | csv_lines = ["template," + (",".join(args.datasets))] 49 | for exp_name_template in args.exp_name_templates: 50 | csv_lines.append(f"{exp_name_template}," + collect_exp_scores(exp_name_template, args.datasets)) 51 | 52 | output_fname = os.path.join(os.getenv("OUTPUT_PATH", default="exp_out"), "summary.csv") 53 | with open(output_fname, "w") as f: 54 | for line in csv_lines: 55 | f.write(line + "\n") 56 | print(f"Save result to {output_fname}") 57 | 58 | 59 | if __name__ == "__main__": 60 | parser = argparse.ArgumentParser() 61 | parser.add_argument("-e", "--exp_name_templates", default="t03b_*_finetune", required=True) 62 | parser.add_argument( 63 | "-d", "--datasets", default="copa,h-swag,storycloze,winogrande,wsc,wic,rte,cb,anli-r1,anli-r2,anli-r3" 64 | ) 65 | args = parser.parse_args() 66 | args.exp_name_templates = args.exp_name_templates.split(",") 67 | args.datasets = args.datasets.split(",") 68 | make_result_table(args) 69 | -------------------------------------------------------------------------------- /src/ticl/model.py: -------------------------------------------------------------------------------- 1 | import torch.nn.functional as F 2 | import torch 3 | 4 | 5 | class InferenceModel: 6 | def __init__(self, model, tokenizer, length_norm, compute_precision, compute_device, compute_batch_size): 7 | self.model = model 8 | self.tokenizer = tokenizer 9 | self.length_norm = length_norm 10 | self.compute_precision = compute_precision 11 | self.compute_device = compute_device 12 | self.compute_batch_size = compute_batch_size 13 | self.model.eval() 14 | self._model_to_device() 15 | 16 | def __call__(self, batch): 17 | with torch.no_grad(): 18 | input_ids, choices_ids = batch["input_ids"], batch["answer_choices_ids"] 19 | input_ids = input_ids.to(self.model.encoder.device) 20 | choices_ids = choices_ids.to(self.model.decoder.device) 21 | 22 | # TODO: use compute_batch_size if CUDA OOM 23 | bs, num_choices = choices_ids.size()[:2] 24 | flat_input_ids = input_ids.flatten(0, 1) 25 | choices_sharing_inputs = input_ids.size(1) != num_choices 26 | 27 | attention_mask = (flat_input_ids != self.tokenizer.pad_token_id).float() 28 | # (bs * (1 or num_choices), max_seq_len) 29 | encoder_hidden_states = self.model.encoder(input_ids=flat_input_ids, attention_mask=attention_mask)[0] 30 | # (bs * (1 or num_choices), max_seq_len, d_model) 31 | if choices_sharing_inputs: 32 | attention_mask = attention_mask.unsqueeze(dim=1).expand(-1, num_choices, -1).flatten(0, 1) 33 | # (bs * num_choices, max_seq_len) 34 | encoder_hidden_states = ( 35 | encoder_hidden_states.unsqueeze(dim=1).expand(-1, num_choices, -1, -1).flatten(0, 1) 36 | ) 37 | # (bs * num_choices, max_seq_len, d_model) 38 | 39 | flat_choices_ids = choices_ids.flatten(0, 1) 40 | decoder_input_ids = torch.cat([torch.zeros_like(flat_choices_ids[:, :1]), flat_choices_ids[:, :-1]], dim=1) 41 | lm_target = flat_choices_ids - 100 * (flat_choices_ids == self.tokenizer.pad_token_id).long() 42 | # (bs * num_choices, max_seq_len) 43 | 44 | model_output = self.model( 45 | attention_mask=attention_mask, 46 | encoder_outputs=[encoder_hidden_states], 47 | decoder_input_ids=decoder_input_ids, 48 | ) 49 | choices_scores = ( 50 | F.cross_entropy(model_output.logits.flatten(0, 1), lm_target.flatten(0, 1), reduction="none") 51 | .view(bs, num_choices, -1) 52 | .sum(dim=-1) 53 | ) 54 | if self.length_norm: 55 | choices_scores = choices_scores / (choices_ids != self.tokenizer.pad_token_id).sum(dim=-1) 56 | # (bs, num_choices) 57 | 58 | return choices_scores.cpu() 59 | 60 | def _model_to_device(self): 61 | if self.compute_precision == "32": 62 | pass 63 | elif self.compute_precision == "16": 64 | self.model.to(torch.float16) 65 | elif self.compute_precision == "bf16": 66 | self.model.to(torch.bfloat16) 67 | 68 | self.model.to(self.compute_device) 69 | if self.compute_device == "cuda" and torch.cuda.device_count() > 1: 70 | self.model.parallelize() 71 | -------------------------------------------------------------------------------- /src/ticl/test_icl.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | from transformers import AutoTokenizer, AutoModelForSeq2SeqLM 4 | import torch.multiprocessing 5 | 6 | 7 | from src.data import get_dataset_reader 8 | from src.utils.util import set_seeds 9 | from src.ticl.icl_engines import get_icl_engine 10 | from src.ticl.model import InferenceModel 11 | 12 | torch.multiprocessing.set_sharing_strategy("file_system") 13 | 14 | 15 | def get_transformer(ticl_config): 16 | tokenizer = AutoTokenizer.from_pretrained(ticl_config.pretrained_model) 17 | model = AutoModelForSeq2SeqLM.from_pretrained(ticl_config.pretrained_model, low_cpu_mem_usage=True) 18 | 19 | tokenizer.model_max_length = ticl_config.max_seq_len 20 | return tokenizer, model 21 | 22 | 23 | def main(ticl_config): 24 | """ 25 | Trains the model 26 | 27 | :param config: 28 | :return: 29 | """ 30 | 31 | print("Load Model") 32 | tokenizer, model = get_transformer(ticl_config) 33 | model = InferenceModel( 34 | model=model, 35 | tokenizer=tokenizer, 36 | length_norm=ticl_config.length_norm, 37 | compute_precision=ticl_config.compute_precision, 38 | compute_device=ticl_config.compute_device, 39 | compute_batch_size=ticl_config.compute_batch_size, 40 | ) 41 | 42 | print("Prepare ICL Data") 43 | dataset_reader = get_dataset_reader(ticl_config) 44 | set_seeds(ticl_config.a_number_between_1_and_100) 45 | icl_engine = get_icl_engine(ticl_config, dataset_reader, tokenizer) 46 | 47 | print("Start Eval") 48 | icl_engine.run(model) 49 | 50 | 51 | if __name__ == "__main__": 52 | parser = argparse.ArgumentParser() 53 | parser.add_argument("--exp_name", type=str, default="debug") 54 | 55 | parser.add_argument("--pretrained_model", type=str, default="google/t5-xxl-lm-adapt") 56 | parser.add_argument("--a_number_between_1_and_100", type=int, default=62) 57 | parser.add_argument("--length_norm", type=str, choices=["True", "False"], default="True") 58 | parser.add_argument("--icl_method", type=str, choices=["concat", "ensemble"], default="ensemble") 59 | parser.add_argument("--icl_modeling", type=str, choices=["direct", "calibrated", "channel"], default="direct") 60 | 61 | parser.add_argument("--dataset", type=str, required=True) 62 | parser.add_argument("--num_shot", type=int, default=32) 63 | parser.add_argument("--num_workers", type=int, default=8) 64 | parser.add_argument("--eval_batch_size", type=int, default=1) 65 | parser.add_argument("--train_template_idx", type=int, default=-1) 66 | parser.add_argument("--eval_template_idx", type=int, default=-1) 67 | parser.add_argument("--max_seq_len", type=int, default=256) 68 | parser.add_argument("--max_context_len", type=int, default=8192) 69 | parser.add_argument("--context_start", type=str, default="") 70 | parser.add_argument("--context_input_target_separator", type=str, default="") 71 | parser.add_argument("--context_example_separator", type=str, default="") 72 | parser.add_argument("--context_end", type=str, default="") 73 | 74 | parser.add_argument("--compute_precision", type=str, choices=["16", "32", "bf16"], default="bf16") 75 | parser.add_argument("--compute_device", type=str, choices=["cpu", "cuda"], default="cuda") 76 | parser.add_argument("--compute_batch_size", type=int, default=128) # Not used at present 77 | 78 | ticl_config = parser.parse_args() 79 | 80 | ticl_config.length_norm = ticl_config.length_norm == "True" 81 | ticl_config.exp_dir = os.path.join(os.getenv("OUTPUT_PATH", default="exp_out"), ticl_config.exp_name) 82 | ticl_config.dev_pred_file = os.path.join(ticl_config.exp_dir, "dev_pred.txt") 83 | ticl_config.dev_score_file = os.path.join(ticl_config.exp_dir, "dev_scores.json") 84 | ticl_config.finish_flag_file = os.path.join(ticl_config.exp_dir, "exp_completed.txt") 85 | ticl_config.change_hswag_templates = False 86 | if not os.path.exists(ticl_config.exp_dir): 87 | os.makedirs(ticl_config.exp_dir) 88 | 89 | print(ticl_config) 90 | 91 | if os.path.exists(ticl_config.finish_flag_file): 92 | print(f"Skip finished experiment {ticl_config.exp_name}") 93 | else: 94 | with open(os.path.join(ticl_config.exp_dir, os.path.join("config.txt")), "w") as f: 95 | f.write(ticl_config.__str__()) 96 | 97 | print(f"Mark experiment {ticl_config.exp_name} as claimed") 98 | with open(ticl_config.finish_flag_file, "w") as f: 99 | f.write("0") 100 | main(ticl_config) 101 | -------------------------------------------------------------------------------- /src/utils/Config.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import ast 4 | 5 | 6 | class Config(object): 7 | def __init__(self, filenames=None, kwargs=None): 8 | # Experiment configs 9 | self.exp_dir = None 10 | self.exp_name = None 11 | self.allow_skip_exp = True 12 | self.seed = 42 13 | 14 | # Model Configs 15 | self.model = "EncDec" 16 | self.max_seq_len = 256 17 | self.origin_model = "bigscience/T0_3B" 18 | self.load_weight = "" 19 | 20 | # Dataset Configs 21 | self.dataset = "rte" 22 | self.few_shot = True 23 | self.num_shot = 16 24 | self.few_shot_random_seed = 100 25 | self.train_template_idx = -1 26 | self.eval_template_idx = -1 27 | self.batch_size = 8 28 | self.eval_batch_size = 16 29 | self.num_workers = 8 30 | self.change_hswag_templates = False 31 | self.raft_cross_validation = True 32 | self.raft_validation_start = 0 33 | self.raft_labels_in_input_string = "comma" 34 | self.cleaned_answer_choices_b77 = False 35 | 36 | # Compute backend configs 37 | self.compute_precision = "bf16" 38 | self.compute_strategy = "none" 39 | 40 | # Trainer configs 41 | self.num_steps = 300 42 | self.eval_epoch_interval = 10_000 43 | self.eval_before_training = True 44 | self.save_model = True 45 | self.save_step_interval = 20_000 46 | self.mc_loss = 0 47 | self.unlikely_loss = 0 48 | self.length_norm = 0 49 | self.grad_accum_factor = 1 50 | self.split_option_at_inference = False # Whether to split the answer choices during eval to lower memory usage for datasets with lots of answer choices 51 | 52 | # Optimization configs 53 | self.optimizer = "adafactor" 54 | self.lr = 3e-4 55 | self.trainable_param_names = ".*" 56 | self.scheduler = "linear_decay_with_warmup" 57 | self.warmup_ratio = 0.06 58 | self.weight_decay = 0 59 | self.scale_parameter = True 60 | self.grad_clip_norm = 1 61 | 62 | # PEFT method configs 63 | self.model_modifier = "" 64 | # Prompt Tuning configs 65 | self.prompt_tuning_num_prefix_emb = 100 66 | self.prompt_tuning_encoder = True 67 | self.prompt_tuning_decoder = True 68 | # LoRA configs 69 | self.lora_rank = 4 70 | self.lora_scaling_rank = 0 71 | self.lora_init_scale = 0.01 72 | self.lora_modules = "none" 73 | self.lora_layers = "none" 74 | # BitFit configs 75 | self.bitfit_modules = ".*" 76 | self.bitfit_layers = "q|k|v|o|wi_[01]|w_o" 77 | # Adapter configs 78 | self.adapter_type = "normal" 79 | self.adapter_non_linearity = "relu" 80 | self.adapter_reduction_factor = 4 81 | self.normal_adapter_residual = True 82 | self.lowrank_adapter_w_init = "glorot-uniform" 83 | self.lowrank_adapter_rank = 1 84 | self.compacter_hypercomplex_division = 8 85 | self.compacter_learn_phm = True 86 | self.compacter_hypercomplex_nonlinearity = "glorot-uniform" # wait, is this really the right name? 87 | self.compacter_shared_phm_rule = False 88 | self.compacter_factorized_phm = False 89 | self.compacter_shared_W_phm = False 90 | self.compacter_factorized_phm_rule = False 91 | self.compacter_phm_c_init = "normal" 92 | self.compacter_phm_rank = 1 93 | self.compacter_phm_init_range = 0.01 94 | self.compacter_kronecker_prod = False 95 | self.compacter_add_compacter_in_self_attention = False 96 | self.compacter_add_compacter_in_cross_attention = False 97 | # Intrinsic SAID configs 98 | self.intrinsic_projection = "fastfood" 99 | self.intrinsic_said = True 100 | self.intrinsic_dim = 2000 101 | self.intrinsic_device = "cpu" 102 | # FISH mask configs 103 | self.fishmask_mode = None 104 | self.fishmask_path = None 105 | self.fishmask_keep_ratio = 0.05 106 | # Prefix Tuning configs 107 | self.prefix_tuning_num_input_tokens = 10 108 | self.prefix_tuning_num_target_tokens = 10 109 | self.prefix_tuning_init_path = None 110 | self.prefix_tuning_init_text = None 111 | self.prefix_tuning_parameterization = "mlp-512" 112 | 113 | if filenames: 114 | for filename in filenames.split("+"): 115 | if not os.path.exists(filename): 116 | filename = os.path.join(os.getenv("CONFIG_PATH", default="configs"), filename) 117 | 118 | self.update_kwargs(json.load(open(filename)), eval=False) 119 | if kwargs: 120 | self.update_kwargs(kwargs) 121 | 122 | self.set_exp_dir() 123 | 124 | def update_kwargs(self, kwargs, eval=True): 125 | for (k, v) in kwargs.items(): 126 | if eval: 127 | try: 128 | v = ast.literal_eval(v) 129 | except ValueError: 130 | v = v 131 | else: 132 | v = v 133 | if not hasattr(self, k): 134 | raise ValueError(f"{k} is not in the config") 135 | setattr(self, k, v) 136 | 137 | def set_exp_dir(self): 138 | """ 139 | Updates the config default values based on parameters passed in from config file 140 | """ 141 | 142 | if self.exp_name is not None: 143 | self.exp_dir = os.path.join(os.getenv("OUTPUT_PATH", default="exp_out"), self.exp_name) 144 | else: 145 | self.exp_dir = os.getenv("OUTPUT_PATH", default="exp_out") 146 | if not os.path.exists(self.exp_dir): 147 | os.makedirs(self.exp_dir) 148 | 149 | if self.exp_dir is not None: 150 | self.train_pred_file = os.path.join(self.exp_dir, "train_pred.txt") 151 | self.dev_pred_file = os.path.join(self.exp_dir, "dev_pred.txt") 152 | self.dev_score_file = os.path.join(self.exp_dir, "dev_scores.json") 153 | self.test_pred_file = os.path.join(self.exp_dir, "test_pred.txt") 154 | self.test_score_file = os.path.join(self.exp_dir, "test_scores.json") 155 | self.save_config(os.path.join(self.exp_dir, os.path.join("config.json"))) 156 | self.finish_flag_file = os.path.join(self.exp_dir, "exp_completed.txt") 157 | 158 | def to_json(self): 159 | """ 160 | Converts parameter values in config to json 161 | :return: json 162 | """ 163 | return json.dumps(self.__dict__, indent=4, sort_keys=False) 164 | 165 | def save_config(self, filename): 166 | """ 167 | Saves the config 168 | """ 169 | with open(filename, "w+") as fout: 170 | fout.write(self.to_json()) 171 | fout.write("\n") 172 | -------------------------------------------------------------------------------- /src/utils/get_optimizer.py: -------------------------------------------------------------------------------- 1 | import torch.optim as optim 2 | from transformers import Adafactor 3 | import re 4 | from collections import defaultdict 5 | 6 | 7 | def get_optimizer(model, config): 8 | """ 9 | Construct optimizer based on config 10 | 11 | :param model: 12 | :param config: 13 | :return: 14 | """ 15 | optim_name = config.optimizer 16 | 17 | def param_name_to_group_name(param_name): 18 | if False: 19 | return ".".join(param_name.split(".")[:3]) 20 | # only needed when the model has many trainable parameters, disabled in our expeirments 21 | else: 22 | return "." 23 | 24 | param_groups = defaultdict(lambda: {"params": []}) 25 | trainable_param_names = set() 26 | for (param_name, param) in model.named_parameters(): 27 | if re.fullmatch(config.trainable_param_names, param_name): 28 | param_groups[param_name_to_group_name(param_name)]["params"].append(param) 29 | trainable_param_names.add(param_name) 30 | else: 31 | param.requires_grad = False 32 | 33 | param_groups = param_groups.values() 34 | if optim_name.lower() == "adam": 35 | optimizer = optim.Adam(param_groups, lr=config.lr) 36 | elif optim_name.lower() == "sgd": 37 | optimizer = optim.SGD(param_groups, lr=config.lr, weight_decay=config.weight_decay) 38 | elif optim_name.lower() == "adamw": 39 | optimizer = optim.AdamW(param_groups, lr=config.lr, weight_decay=config.weight_decay, eps=1e-8) 40 | elif optim_name.lower() == "adafactor": 41 | optimizer = Adafactor( 42 | param_groups, 43 | lr=config.lr, 44 | weight_decay=config.weight_decay, 45 | scale_parameter=config.scale_parameter, 46 | relative_step=False, 47 | warmup_init=False, 48 | ) 49 | else: 50 | raise ValueError("Invalid Optimizer name %s" % optim_name) 51 | 52 | return optimizer, trainable_param_names 53 | -------------------------------------------------------------------------------- /src/utils/get_scheduler.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.optim.lr_scheduler import LambdaLR 3 | from transformers.optimization import AdafactorSchedule 4 | 5 | 6 | def get_scheduler(optimizer, config): 7 | """ 8 | Get scheduler 9 | 10 | :param optimizer: 11 | :param config: 12 | :return: 13 | """ 14 | scheduler_name = config.scheduler 15 | num_warmup_steps = config.num_steps * config.warmup_ratio 16 | 17 | if scheduler_name == "polynomial_decay_with_warmup": 18 | return get_polynomial_decay_schedule_with_warmup(optimizer, num_warmup_steps, config.num_steps) 19 | elif scheduler_name == "exponential_decay": 20 | return torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=config.gamma) 21 | elif scheduler_name == "linear_decay_with_warmup": 22 | return get_linear_schedule_with_warmup(optimizer, num_warmup_steps, config.num_steps) 23 | elif scheduler_name == "cosine_annealing": 24 | return torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, config.num_steps) 25 | elif scheduler_name == "adafactor": 26 | return AdafactorSchedule(optimizer, config.lr) 27 | else: 28 | raise ValueError("Invalid Scheduler Name %s" % scheduler_name) 29 | 30 | 31 | def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1): 32 | """ 33 | Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, 34 | after a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer. 35 | 36 | Args: 37 | optimizer (:class:`~torch.optim.Optimizer`): 38 | The optimizer for which to schedule the learning rate. 39 | num_warmup_steps (:obj:`int`): 40 | The number of steps for the warmup phase. 41 | num_training_steps (:obj:`int`): 42 | The total number of training steps. 43 | last_epoch (:obj:`int`, `optional`, defaults to -1): 44 | The index of the last epoch when resuming training. 45 | 46 | Return: 47 | :obj:`torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. 48 | """ 49 | 50 | def lr_lambda(current_step: int): 51 | if current_step < num_warmup_steps: 52 | return float(current_step) / float(max(1, num_warmup_steps)) 53 | return max(0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))) 54 | 55 | return LambdaLR(optimizer, lr_lambda, last_epoch) 56 | 57 | 58 | def get_polynomial_decay_schedule_with_warmup( 59 | optimizer, num_warmup_steps, num_training_steps, lr_end=1e-7, power=1.0, last_epoch=-1 60 | ): 61 | """ 62 | Create a schedule with a learning rate that decreases as a polynomial decay 63 | from the initial lr set in the optimizer to end lr defined by `lr_end`, 64 | after a warmup period during which it increases linearly from 0 to the 65 | initial lr set in the optimizer. 66 | 67 | Args: 68 | optimizer (:class:`~torch.optim.Optimizer`): 69 | The optimizer for which to schedule the learning rate. 70 | num_warmup_steps (:obj:`int`): 71 | The number of steps for the warmup phase. 72 | num_training_steps (:obj:`int`): 73 | The total number of training steps. 74 | lr_end (:obj:`float`, `optional`, defaults to 1e-7): 75 | The end LR. 76 | power (:obj:`float`, `optional`, defaults to 1.0): 77 | Power factor. 78 | last_epoch (:obj:`int`, `optional`, defaults to -1): 79 | The index of the last epoch when resuming training. 80 | 81 | Note: `power` defaults to 1.0 as in the fairseq implementation, which in turn is 82 | based on the original BERT implementation at 83 | https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37 84 | 85 | Return: 86 | :obj:`torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. 87 | 88 | """ 89 | 90 | lr_init = optimizer.defaults["lr"] 91 | assert lr_init > lr_end, f"lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})" 92 | 93 | def lr_lambda(current_step: int): 94 | if current_step < num_warmup_steps: 95 | return float(current_step) / float(max(1, num_warmup_steps)) 96 | elif current_step > num_training_steps: 97 | return lr_end / lr_init # as LambdaLR multiplies by lr_init 98 | else: 99 | lr_range = lr_init - lr_end 100 | decay_steps = num_training_steps - num_warmup_steps 101 | pct_remaining = 1 - (current_step - num_warmup_steps) / decay_steps 102 | decay = lr_range * pct_remaining ** power + lr_end 103 | return decay / lr_init # as LambdaLR multiplies by lr_init 104 | 105 | return LambdaLR(optimizer, lr_lambda, last_epoch) 106 | -------------------------------------------------------------------------------- /src/utils/util.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import datetime 3 | import os 4 | import sys 5 | import argparse 6 | import psutil 7 | 8 | import subprocess 9 | from shutil import copytree, ignore_patterns 10 | import random 11 | import numpy as np 12 | import logging 13 | import re 14 | import sys 15 | 16 | global device; device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 17 | 18 | 19 | def my_collate_fn(batch): 20 | 21 | dict_batch = {} 22 | dict_batch["input"] = {} 23 | dict_batch["output"] = {} 24 | 25 | for datapoint in batch: 26 | for (k, v) in datapoint["input"].items(): 27 | if k in dict_batch["input"]: 28 | dict_batch["input"][k].append(v) 29 | # dict_batch["input"][k].append(v[0]) 30 | else: 31 | # dict_batch["input"][k] = [v[0]] 32 | dict_batch["input"][k] = [v] 33 | 34 | 35 | for (k, v) in datapoint["output"].items(): 36 | if k in dict_batch["output"]: 37 | # dict_batch["output"][k].append(v[0]) 38 | dict_batch["output"][k].append(v) 39 | 40 | else: 41 | # dict_batch["output"][k] = [v[0]] 42 | dict_batch["output"][k] = [v] 43 | 44 | 45 | for (k, list_v) in dict_batch["input"].items(): 46 | if isinstance(list_v[0], int): 47 | dict_batch["input"][k] = torch.tensor(list_v) 48 | for (k, list_v) in dict_batch["output"].items(): 49 | if isinstance(list_v[0], int): 50 | dict_batch["output"][k] = torch.tensor(list_v) 51 | 52 | return dict_batch 53 | 54 | def set_global_logging_level(level=logging.ERROR, prefices=[""]): 55 | """ 56 | Override logging levels of different modules based on their name as a prefix. 57 | It needs to be invoked after the modules have been loaded so that their loggers have been initialized. 58 | 59 | Args: 60 | - level: desired level. e.g. logging.INFO. Optional. Default is logging.ERROR 61 | - prefices: list of one or more str prefices to match (e.g. ["transformers", "torch"]). Optional. 62 | Default is `[""]` to match all active loggers. 63 | The match is a case-sensitive `module_name.startswith(prefix)` 64 | """ 65 | prefix_re = re.compile(fr'^(?:{ "|".join(prefices) })') 66 | for name in logging.root.manager.loggerDict: 67 | if re.match(prefix_re, name): 68 | logging.getLogger(name).setLevel(level) 69 | 70 | 71 | def set_seeds(seed): 72 | "set random seeds" 73 | random.seed(seed) 74 | np.random.seed(seed) 75 | torch.manual_seed(seed) 76 | torch.cuda.manual_seed_all(seed) 77 | 78 | def make_dir(dir_name): 79 | ''' 80 | Makes a directory if it doesn't exists yet 81 | Args: 82 | dir_name: directory name 83 | ''' 84 | if not os.path.exists(dir_name): 85 | os.makedirs(dir_name) 86 | 87 | def make_exp_dir(base_exp_dir): 88 | ''' 89 | Makes an experiment directory with timestamp 90 | Args: 91 | base_output_dir_name: base output directory name 92 | Returns: 93 | exp_dir_name: experiment directory name 94 | ''' 95 | now = datetime.datetime.now() 96 | ts = "{:04d}-{:02d}-{:02d}-{:02d}-{:02d}-{:02d}".format(now.year, now.month, now.day, now.hour, now.minute, 97 | now.second) 98 | exp_dir_name = os.path.join(base_exp_dir, ts) 99 | make_dir(exp_dir_name) 100 | 101 | src_file = os.path.join(exp_dir_name, 'src') 102 | 103 | copytree(os.path.join(os.environ['NICL_ROOT'], "src"), src_file, ignore=ignore_patterns('*.pyc', 'tmp*')) 104 | 105 | return exp_dir_name 106 | 107 | 108 | def print_mem_usage(loc): 109 | ''' 110 | Print memory usage in GB 111 | :return: 112 | ''' 113 | print("%s gpu mem allocated: %.2f GB; reserved: %.2f GB; max: %.2f GB; cpu mem %d" \ 114 | % (loc, float(torch.cuda.memory_allocated() / 1e9), float(torch.cuda.memory_reserved() / 1e9), float(torch.cuda.max_memory_allocated() / 1e9), psutil.virtual_memory().percent)) 115 | sys.stdout.flush() 116 | 117 | class ParseKwargs(argparse.Action): 118 | def __call__(self, parser, namespace, values, option_string=None): 119 | setattr(namespace, self.dest, dict()) 120 | for value in values: 121 | key, value = value.split('=') 122 | getattr(namespace, self.dest)[key] = value 123 | 124 | 125 | def update_dict_val_store(dict_val_store, dict_update_val, grad_accum_factor): 126 | ''' 127 | Update dict_val_store with dict_update_val 128 | 129 | :param dict_val_store: 130 | :param dict_update_val: 131 | :return: 132 | ''' 133 | if dict_val_store is None: 134 | dict_val_store = dict_update_val 135 | else: 136 | for k in dict_val_store.keys(): 137 | dict_val_store[k] += dict_update_val[k] / grad_accum_factor 138 | 139 | return dict_val_store 140 | 141 | def get_avg_dict_val_store(dict_val_store, num_batches, grad_accumulation_factor): 142 | ''' 143 | Get average dictionary val 144 | 145 | :param dict_val_store: 146 | :param eval_every: 147 | :return: 148 | ''' 149 | dict_avg_val = {} 150 | 151 | for k in dict_val_store.keys(): 152 | dict_avg_val[k] = float('%.3f' % (dict_val_store[k] / num_batches / grad_accumulation_factor)) 153 | 154 | return dict_avg_val --------------------------------------------------------------------------------