├── .dockerignore ├── .gitignore ├── Dockerfile ├── LICENSE.txt ├── README.md ├── attribution ├── __init__.py ├── drop_model.py ├── drop_reader.py ├── huggingface_readers.py ├── ia3.py ├── icl_readers.py ├── mam.py ├── model.py ├── ni_model.py ├── ni_reader.py ├── p3_cluster_reader.py ├── p3_jsonl_reader.py └── qasper_reader.py ├── data ├── README.md ├── eurovoc_descriptors.json ├── few_shot │ ├── anli-r1 │ │ └── 50_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── anli-r2 │ │ └── 50_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── anli-r3 │ │ └── 50_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── boolq │ │ └── 32_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── casehold │ │ └── 32_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 100_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── cb │ │ └── 32_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── copa │ │ └── 32_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 100_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── h-swag │ │ └── 20_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── rte │ │ └── 32_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 100_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── sst2 │ │ └── 16_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── storycloze │ │ └── 70_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── wic │ │ └── 32_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 100_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ ├── winogrande │ │ └── 50_shot │ │ │ ├── 0_seed.jsonl │ │ │ ├── 1024_seed.jsonl │ │ │ ├── 1_seed.jsonl │ │ │ ├── 32_seed.jsonl │ │ │ └── 42_seed.jsonl │ └── wsc │ │ └── 32_shot │ │ ├── 0_seed.jsonl │ │ ├── 1024_seed.jsonl │ │ ├── 1_seed.jsonl │ │ ├── 32_seed.jsonl │ │ └── 42_seed.jsonl ├── qasper │ ├── qasper-dev-eval-v0.3.json │ └── qasper-test-v0.3.json └── t0_prompt_tasks.txt ├── figure.png ├── natural_instructions ├── ni_collator.py ├── ni_dataset.py ├── ni_evaluation.py ├── ni_reference.jsonl └── task_eval_splits.py ├── requirements.txt ├── scripts ├── add_options.py ├── analyze_surface_similarity.py ├── compute_stats.py ├── construct_balanced_sample.py ├── convert_allennlp_pred_to_drop_eval_format.py ├── convert_pyserini.py ├── download_p3.py ├── drop_eval_script.py ├── dump_dataset_queries.py ├── evaluate_eurlex_preds.py ├── evaluate_qasper_evidence_predictions.py ├── evaluate_unfair_tos_preds.py ├── generate_natural_instructions_query_splits.py ├── get_random_indices.py ├── grab_instance_idxes.py ├── group_instances_without_prompts.py ├── index_ni_train_reps.py ├── index_p3_train_reps.py ├── indices_to_file.py ├── infer_prompts.py ├── make_final_rep_clusters.py ├── make_gradient_clusters.py ├── make_tf_idf_clusters.py ├── parse_pyserini_output.py ├── reduce_indices_to_balanced.py ├── retrieve_training_data.py ├── retrieve_training_data_ni.py ├── retrieve_training_data_rank.py ├── select_few_shots.py ├── show_clusters.py ├── simplify_p3_data.py ├── sort_training_data.py └── write_p3_train_instances.py ├── shell_scripts ├── create_pyserini_index.sh ├── evaluate_drop.sh ├── evaluate_natural_instructions.sh ├── evaluate_qasper.sh └── retrieve_pyserini.sh └── training_config ├── drop_train.jsonnet ├── ia3_train.jsonnet ├── ia3_train_load.jsonnet ├── qasper_train.jsonnet ├── t5_small_cluster_train.jsonnet ├── t5_small_dump_without_training.jsonnet ├── t5_small_gradient_cluster_train.jsonnet ├── t5_xl_cluster_train.jsonnet ├── task_multitask_train.jsonnet ├── task_retrieved_only.jsonnet └── task_specific_train.jsonnet /.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/.dockerignore -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/.gitignore -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/LICENSE.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/README.md -------------------------------------------------------------------------------- /attribution/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /attribution/drop_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/drop_model.py -------------------------------------------------------------------------------- /attribution/drop_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/drop_reader.py -------------------------------------------------------------------------------- /attribution/huggingface_readers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/huggingface_readers.py -------------------------------------------------------------------------------- /attribution/ia3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/ia3.py -------------------------------------------------------------------------------- /attribution/icl_readers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/icl_readers.py -------------------------------------------------------------------------------- /attribution/mam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/mam.py -------------------------------------------------------------------------------- /attribution/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/model.py -------------------------------------------------------------------------------- /attribution/ni_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/ni_model.py -------------------------------------------------------------------------------- /attribution/ni_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/ni_reader.py -------------------------------------------------------------------------------- /attribution/p3_cluster_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/p3_cluster_reader.py -------------------------------------------------------------------------------- /attribution/p3_jsonl_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/p3_jsonl_reader.py -------------------------------------------------------------------------------- /attribution/qasper_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/attribution/qasper_reader.py -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/README.md -------------------------------------------------------------------------------- /data/eurovoc_descriptors.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/eurovoc_descriptors.json -------------------------------------------------------------------------------- /data/few_shot/anli-r1/50_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r1/50_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r1/50_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r1/50_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r1/50_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r1/50_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r1/50_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r1/50_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r1/50_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r1/50_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r2/50_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r2/50_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r2/50_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r2/50_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r2/50_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r2/50_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r2/50_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r2/50_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r2/50_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r2/50_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r3/50_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r3/50_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r3/50_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r3/50_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r3/50_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r3/50_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r3/50_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r3/50_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/anli-r3/50_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/anli-r3/50_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/boolq/32_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/boolq/32_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/boolq/32_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/boolq/32_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/boolq/32_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/boolq/32_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/boolq/32_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/boolq/32_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/boolq/32_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/boolq/32_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/casehold/32_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/casehold/32_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/casehold/32_shot/100_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/casehold/32_shot/100_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/casehold/32_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/casehold/32_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/casehold/32_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/casehold/32_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/casehold/32_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/casehold/32_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/cb/32_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/cb/32_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/cb/32_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/cb/32_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/cb/32_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/cb/32_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/cb/32_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/cb/32_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/cb/32_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/cb/32_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/copa/32_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/100_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/copa/32_shot/100_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/copa/32_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/copa/32_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/copa/32_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/copa/32_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/copa/32_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/h-swag/20_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/h-swag/20_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/h-swag/20_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/h-swag/20_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/h-swag/20_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/h-swag/20_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/h-swag/20_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/h-swag/20_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/h-swag/20_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/h-swag/20_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/rte/32_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/rte/32_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/rte/32_shot/100_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/rte/32_shot/100_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/rte/32_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/rte/32_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/rte/32_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/rte/32_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/rte/32_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/rte/32_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/rte/32_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/rte/32_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/sst2/16_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/sst2/16_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/sst2/16_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/sst2/16_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/sst2/16_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/sst2/16_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/sst2/16_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/sst2/16_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/sst2/16_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/sst2/16_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/storycloze/70_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/storycloze/70_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/storycloze/70_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/storycloze/70_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/storycloze/70_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/storycloze/70_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/storycloze/70_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/storycloze/70_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/storycloze/70_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/storycloze/70_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/wic/32_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/100_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/wic/32_shot/100_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/wic/32_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/wic/32_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/wic/32_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/wic/32_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/wic/32_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/winogrande/50_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/winogrande/50_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/winogrande/50_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/winogrande/50_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/winogrande/50_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/winogrande/50_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/winogrande/50_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/winogrande/50_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/winogrande/50_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/winogrande/50_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/wsc/32_shot/0_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/wsc/32_shot/0_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/wsc/32_shot/1024_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/wsc/32_shot/1024_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/wsc/32_shot/1_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/wsc/32_shot/1_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/wsc/32_shot/32_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/wsc/32_shot/32_seed.jsonl -------------------------------------------------------------------------------- /data/few_shot/wsc/32_shot/42_seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/few_shot/wsc/32_shot/42_seed.jsonl -------------------------------------------------------------------------------- /data/qasper/qasper-dev-eval-v0.3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/qasper/qasper-dev-eval-v0.3.json -------------------------------------------------------------------------------- /data/qasper/qasper-test-v0.3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/qasper/qasper-test-v0.3.json -------------------------------------------------------------------------------- /data/t0_prompt_tasks.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/data/t0_prompt_tasks.txt -------------------------------------------------------------------------------- /figure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/figure.png -------------------------------------------------------------------------------- /natural_instructions/ni_collator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/natural_instructions/ni_collator.py -------------------------------------------------------------------------------- /natural_instructions/ni_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/natural_instructions/ni_dataset.py -------------------------------------------------------------------------------- /natural_instructions/ni_evaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/natural_instructions/ni_evaluation.py -------------------------------------------------------------------------------- /natural_instructions/ni_reference.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/natural_instructions/ni_reference.jsonl -------------------------------------------------------------------------------- /natural_instructions/task_eval_splits.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/natural_instructions/task_eval_splits.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/add_options.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/add_options.py -------------------------------------------------------------------------------- /scripts/analyze_surface_similarity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/analyze_surface_similarity.py -------------------------------------------------------------------------------- /scripts/compute_stats.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/compute_stats.py -------------------------------------------------------------------------------- /scripts/construct_balanced_sample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/construct_balanced_sample.py -------------------------------------------------------------------------------- /scripts/convert_allennlp_pred_to_drop_eval_format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/convert_allennlp_pred_to_drop_eval_format.py -------------------------------------------------------------------------------- /scripts/convert_pyserini.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/convert_pyserini.py -------------------------------------------------------------------------------- /scripts/download_p3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/download_p3.py -------------------------------------------------------------------------------- /scripts/drop_eval_script.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/drop_eval_script.py -------------------------------------------------------------------------------- /scripts/dump_dataset_queries.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/dump_dataset_queries.py -------------------------------------------------------------------------------- /scripts/evaluate_eurlex_preds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/evaluate_eurlex_preds.py -------------------------------------------------------------------------------- /scripts/evaluate_qasper_evidence_predictions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/evaluate_qasper_evidence_predictions.py -------------------------------------------------------------------------------- /scripts/evaluate_unfair_tos_preds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/evaluate_unfair_tos_preds.py -------------------------------------------------------------------------------- /scripts/generate_natural_instructions_query_splits.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/generate_natural_instructions_query_splits.py -------------------------------------------------------------------------------- /scripts/get_random_indices.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/get_random_indices.py -------------------------------------------------------------------------------- /scripts/grab_instance_idxes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/grab_instance_idxes.py -------------------------------------------------------------------------------- /scripts/group_instances_without_prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/group_instances_without_prompts.py -------------------------------------------------------------------------------- /scripts/index_ni_train_reps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/index_ni_train_reps.py -------------------------------------------------------------------------------- /scripts/index_p3_train_reps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/index_p3_train_reps.py -------------------------------------------------------------------------------- /scripts/indices_to_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/indices_to_file.py -------------------------------------------------------------------------------- /scripts/infer_prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/infer_prompts.py -------------------------------------------------------------------------------- /scripts/make_final_rep_clusters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/make_final_rep_clusters.py -------------------------------------------------------------------------------- /scripts/make_gradient_clusters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/make_gradient_clusters.py -------------------------------------------------------------------------------- /scripts/make_tf_idf_clusters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/make_tf_idf_clusters.py -------------------------------------------------------------------------------- /scripts/parse_pyserini_output.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/parse_pyserini_output.py -------------------------------------------------------------------------------- /scripts/reduce_indices_to_balanced.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/reduce_indices_to_balanced.py -------------------------------------------------------------------------------- /scripts/retrieve_training_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/retrieve_training_data.py -------------------------------------------------------------------------------- /scripts/retrieve_training_data_ni.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/retrieve_training_data_ni.py -------------------------------------------------------------------------------- /scripts/retrieve_training_data_rank.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/retrieve_training_data_rank.py -------------------------------------------------------------------------------- /scripts/select_few_shots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/select_few_shots.py -------------------------------------------------------------------------------- /scripts/show_clusters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/show_clusters.py -------------------------------------------------------------------------------- /scripts/simplify_p3_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/simplify_p3_data.py -------------------------------------------------------------------------------- /scripts/sort_training_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/sort_training_data.py -------------------------------------------------------------------------------- /scripts/write_p3_train_instances.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/scripts/write_p3_train_instances.py -------------------------------------------------------------------------------- /shell_scripts/create_pyserini_index.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/shell_scripts/create_pyserini_index.sh -------------------------------------------------------------------------------- /shell_scripts/evaluate_drop.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/shell_scripts/evaluate_drop.sh -------------------------------------------------------------------------------- /shell_scripts/evaluate_natural_instructions.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/shell_scripts/evaluate_natural_instructions.sh -------------------------------------------------------------------------------- /shell_scripts/evaluate_qasper.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/shell_scripts/evaluate_qasper.sh -------------------------------------------------------------------------------- /shell_scripts/retrieve_pyserini.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/shell_scripts/retrieve_pyserini.sh -------------------------------------------------------------------------------- /training_config/drop_train.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/training_config/drop_train.jsonnet -------------------------------------------------------------------------------- /training_config/ia3_train.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/training_config/ia3_train.jsonnet -------------------------------------------------------------------------------- /training_config/ia3_train_load.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/training_config/ia3_train_load.jsonnet -------------------------------------------------------------------------------- /training_config/qasper_train.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/training_config/qasper_train.jsonnet -------------------------------------------------------------------------------- /training_config/t5_small_cluster_train.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/training_config/t5_small_cluster_train.jsonnet -------------------------------------------------------------------------------- /training_config/t5_small_dump_without_training.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/training_config/t5_small_dump_without_training.jsonnet -------------------------------------------------------------------------------- /training_config/t5_small_gradient_cluster_train.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/training_config/t5_small_gradient_cluster_train.jsonnet -------------------------------------------------------------------------------- /training_config/t5_xl_cluster_train.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/training_config/t5_xl_cluster_train.jsonnet -------------------------------------------------------------------------------- /training_config/task_multitask_train.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/training_config/task_multitask_train.jsonnet -------------------------------------------------------------------------------- /training_config/task_retrieved_only.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/training_config/task_retrieved_only.jsonnet -------------------------------------------------------------------------------- /training_config/task_specific_train.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/data-efficient-finetuning/HEAD/training_config/task_specific_train.jsonnet --------------------------------------------------------------------------------