├── .gitignore ├── README.md ├── bart.py ├── cli_maml.py ├── cli_multitask.py ├── cli_singletask.py ├── dataloader ├── custom_tasks_splits │ ├── random.json │ ├── train_classification_test_classification.json │ ├── train_halfhalf_test_classification.json │ ├── train_non_mrc_qa_test_mrc.json │ ├── train_non_multiple_choice_qa_test_multiple_choice.json │ ├── train_non_nli_test_nli.json │ ├── train_nonclassification_test_classification.json │ └── train_nonparaphrase_classification_test_paraphrase.json ├── fewshot_gym_metalearn.py ├── fewshot_gym_multitask.py ├── fewshot_gym_singletask.py ├── metrics.py └── utils.py ├── example_scripts ├── collect_results.py ├── finetune_a_list_of_tasks.sh ├── finetune_boolq.sh ├── finetune_boolq_tune_hp.sh ├── upstream_maml.sh └── upstream_multitask.sh ├── run_maml.py ├── run_multitask.py ├── run_singletask.py ├── tasks ├── _all_tasks.py ├── _build_gym.py ├── _md5sum.py ├── acronym_identification.py ├── ade_classification.py ├── ade_dosage.py ├── ade_effect.py ├── adversarial_qa.py ├── aeslc.py ├── agnews.py ├── ai2_arc.py ├── amazon_polarity.py ├── anli.py ├── app_reviews.py ├── aqua_rat.py ├── art.py ├── aslg_pc12.py ├── biomrc.py ├── blimp.py ├── boolq.py ├── break.py ├── circa.py ├── climate_fever.py ├── codah.py ├── commongen.py ├── commonsense_qa.py ├── cos_e.py ├── cosmos_qa.py ├── crawl_domain.py ├── crows_pairs.py ├── dbpedia_14.py ├── definite_pronoun_resolution.py ├── discovery.py ├── dream.py ├── duorc.py ├── e2e_nlg_cleaned.py ├── eli5.py ├── emo.py ├── emotion.py ├── empathetic_dialogues.py ├── ethos.py ├── fewshot_gym_dataset.py ├── financial_phrasebank.py ├── freebase_qa.py ├── gigaword.py ├── glue_cola.py ├── glue_mnli.py ├── glue_mrpc.py ├── glue_qnli.py ├── glue_qqp.py ├── glue_rte.py ├── glue_sst2.py ├── glue_wnli.py ├── google_wellformed_query.py ├── hate_speech18.py ├── hate_speech_offensive.py ├── hatexplain.py ├── health_fact.py ├── hellaswag.py ├── hotpot_qa.py ├── imdb.py ├── jeopardy.py ├── kilt_ay2.py ├── kilt_fever.py ├── kilt_hotpotqa.py ├── kilt_nq.py ├── kilt_trex.py ├── kilt_wow.py ├── kilt_zsre.py ├── lama.py ├── liar.py ├── limit.py ├── math_qa.py ├── mc_taco.py ├── medical_questions_pairs.py ├── mocha.py ├── multi_news.py ├── not_used │ ├── 0ambig_qa.py │ ├── 0bc2gm_corpus.py │ ├── 0cnn_dailymail.py │ ├── 0conll2003.py │ ├── 0fever.py │ ├── 0jnlpba.py │ ├── 0kilt_eli5.py │ ├── 0kilt_triviaqa.py │ ├── 0linnaeus.py │ ├── 0math_dataset.py │ ├── 0ncbi_disease.py │ ├── 0nq_open.py │ ├── 0web_of_science.py │ ├── 0wiki_movies.py │ └── 0wnut_17.py ├── numer_sense.py ├── onestop_english.py ├── openbookqa.py ├── paws.py ├── piqa.py ├── poem_sentiment.py ├── proto_qa.py ├── qa_srl.py ├── qasc.py ├── quail.py ├── quarel.py ├── quartz.py ├── quoref.py ├── race.py ├── reddit_tifu.py ├── ropes.py ├── rotten_tomatoes.py ├── samsum.py ├── scicite.py ├── sciq.py ├── scitail.py ├── search_qa.py ├── sick.py ├── sms_spam.py ├── social_i_qa.py ├── spider.py ├── squad.py ├── superglue_cb.py ├── superglue_copa.py ├── superglue_multirc.py ├── superglue_record.py ├── superglue_rte.py ├── superglue_wic.py ├── superglue_wsc.py ├── swag.py ├── tab_fact.py ├── trec.py ├── trec_finegrained.py ├── tweet_eval.py ├── tweet_qa.py ├── utils.py ├── web_questions.py ├── wiki_auto.py ├── wiki_bio.py ├── wiki_qa.py ├── wiki_split.py ├── wikisql.py ├── winogrande.py ├── wiqa.py ├── xsum.py ├── yahoo_answers_topics.py ├── yelp_polarity.py └── yelp_review_full.py ├── tune_hps_singletask.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | data/ 2 | daily/ 3 | models/ 4 | tasks/full_datasets 5 | playground/ 6 | data_json/ 7 | tasks_more_shots/ 8 | 9 | 10 | # Byte-compiled / optimized / DLL files 11 | __pycache__/ 12 | *.py[cod] 13 | *$py.class 14 | 15 | # C extensions 16 | *.so 17 | 18 | # Distribution / packaging 19 | .Python 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | wheels/ 32 | pip-wheel-metadata/ 33 | share/python-wheels/ 34 | *.egg-info/ 35 | .installed.cfg 36 | *.egg 37 | MANIFEST 38 | 39 | # PyInstaller 40 | # Usually these files are written by a python script from a template 41 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 42 | *.manifest 43 | *.spec 44 | 45 | # Installer logs 46 | pip-log.txt 47 | pip-delete-this-directory.txt 48 | 49 | # Unit test / coverage reports 50 | htmlcov/ 51 | .tox/ 52 | .nox/ 53 | .coverage 54 | .coverage.* 55 | .cache 56 | nosetests.xml 57 | coverage.xml 58 | *.cover 59 | *.py,cover 60 | .hypothesis/ 61 | .pytest_cache/ 62 | 63 | # Translations 64 | *.mo 65 | *.pot 66 | 67 | # Django stuff: 68 | *.log 69 | local_settings.py 70 | db.sqlite3 71 | db.sqlite3-journal 72 | 73 | # Flask stuff: 74 | instance/ 75 | .webassets-cache 76 | 77 | # Scrapy stuff: 78 | .scrapy 79 | 80 | # Sphinx documentation 81 | docs/_build/ 82 | 83 | # PyBuilder 84 | target/ 85 | 86 | # Jupyter Notebook 87 | .ipynb_checkpoints 88 | 89 | # IPython 90 | profile_default/ 91 | ipython_config.py 92 | 93 | # pyenv 94 | .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 104 | __pypackages__/ 105 | 106 | # Celery stuff 107 | celerybeat-schedule 108 | celerybeat.pid 109 | 110 | # SageMath parsed files 111 | *.sage.py 112 | 113 | # Environments 114 | .env 115 | .venv 116 | env/ 117 | venv/ 118 | ENV/ 119 | env.bak/ 120 | venv.bak/ 121 | 122 | # Spyder project settings 123 | .spyderproject 124 | .spyproject 125 | 126 | # Rope project settings 127 | .ropeproject 128 | 129 | # mkdocs documentation 130 | /site 131 | 132 | # mypy 133 | .mypy_cache/ 134 | .dmypy.json 135 | dmypy.json 136 | 137 | # Pyre type checker 138 | .pyre/ 139 | -------------------------------------------------------------------------------- /bart.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import Tensor, nn 4 | from transformers import T5ForConditionalGeneration, BartForConditionalGeneration 5 | from transformers.modeling_bart import shift_tokens_right 6 | 7 | from utils import label_smoothed_nll_loss 8 | 9 | class MyBart(BartForConditionalGeneration): 10 | def forward(self, input_ids, attention_mask=None, encoder_outputs=None, 11 | decoder_input_ids=None, decoder_attention_mask=None, decoder_cached_states=None, 12 | use_cache=False, is_training=False): 13 | 14 | if is_training: 15 | _decoder_input_ids = shift_tokens_right(decoder_input_ids, self.config.pad_token_id) 16 | else: 17 | _decoder_input_ids = decoder_input_ids 18 | 19 | outputs = self.model( 20 | input_ids, 21 | attention_mask=attention_mask, 22 | encoder_outputs=encoder_outputs, 23 | decoder_input_ids=_decoder_input_ids, 24 | decoder_attention_mask=decoder_attention_mask, 25 | decoder_cached_states=decoder_cached_states, 26 | use_cache=use_cache, 27 | ) 28 | lm_logits = F.linear(outputs[0], self.model.shared.weight, bias=self.final_logits_bias) 29 | if is_training: 30 | lprobs = F.log_softmax(lm_logits, dim=-1) 31 | loss, _ = label_smoothed_nll_loss(lprobs, decoder_input_ids, epsilon=0.1, ignore_index=self.config.pad_token_id) 32 | return loss 33 | return (lm_logits, ) + outputs[1:] 34 | 35 | -------------------------------------------------------------------------------- /dataloader/custom_tasks_splits/train_halfhalf_test_classification.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": ["ade_corpus_v2-dosage", "biomrc", "blimp-ellipsis_n_bar_2", "blimp-sentential_negation_npi_scope", "commonsense_qa", "crows_pairs", "duorc", "hellaswag", "kilt_zsre", "lama-google_re", "lama-squad", "math_qa", "numer_sense", "openbookqa", "piqa", "proto_qa", "quartz-no_knowledge", "race-high", "reddit_tifu-tldr", "ropes", "sciq", "wiki_bio", "discovery", "emotion", "ethos-disability", "ethos-sexual_orientation", "glue-cola", "glue-mnli", "glue-mrpc", "glue-qqp", "glue-rte", "glue-wnli", "hatexplain", "health_fact", "imdb", "paws", "scicite", "sick", "sms_spam", "superglue-rte", "superglue-wsc", "tweet_eval-emotion", "tweet_eval-offensive", "tweet_eval-sentiment", "tweet_eval-stance_hillary"], 3 | "dev": ["tweet_eval-stance_feminist", "ethos-national_origin", "tweet_eval-hate", "ag_news", "amazon_polarity", "hate_speech18", "poem_sentiment", "climate_fever", "medical_questions_pairs", "tweet_eval-stance_atheism"], 4 | "test": ["superglue-cb", "dbpedia_14", "wiki_qa", "emo", "yelp_polarity", "ethos-religion", "financial_phrasebank", "tab_fact", "anli", "ethos-race"] 5 | } -------------------------------------------------------------------------------- /dataloader/custom_tasks_splits/train_non_mrc_qa_test_mrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": [ 3 | "ai2_arc", 4 | "aqua_rat", 5 | "boolq", 6 | "codah", 7 | "commonsense_qa", 8 | "cosmos_qa", 9 | "dream", 10 | "eli5-askh", 11 | "eli5-asks", 12 | "eli5-eli5", 13 | "freebase_qa", 14 | "hellaswag", 15 | "jeopardy", 16 | "kilt_hotpotqa", 17 | "kilt_nq", 18 | "kilt_trex", 19 | "kilt_zsre", 20 | "lama-conceptnet", 21 | "lama-google_re", 22 | "lama-squad", 23 | "lama-trex", 24 | "math_qa", 25 | "mc_taco", 26 | "numer_sense", 27 | "openbookqa", 28 | "qasc", 29 | "quail", 30 | "quarel", 31 | "quartz-no_knowledge", 32 | "quartz-with_knowledge", 33 | "race-high", 34 | "race-middle", 35 | "sciq", 36 | "search_qa", 37 | "social_i_qa", 38 | "squad-no_context", 39 | "superglue-copa", 40 | "superglue-multirc", 41 | "swag", 42 | "web_questions", 43 | "wino_grande", 44 | "wiqa" 45 | ], 46 | "dev": [ 47 | "adversarialqa", 48 | "biomrc", 49 | "duorc", 50 | "hotpot_qa", 51 | "quoref", 52 | "ropes", 53 | "squad-with_context", 54 | "superglue-record", 55 | "tweet_qa" 56 | ], 57 | "test": [] 58 | } -------------------------------------------------------------------------------- /dataloader/custom_tasks_splits/train_non_multiple_choice_qa_test_multiple_choice.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": [ 3 | "adversarialqa", 4 | "biomrc", 5 | "boolq", 6 | "duorc", 7 | "eli5-askh", 8 | "eli5-asks", 9 | "eli5-eli5", 10 | "freebase_qa", 11 | "hotpot_qa", 12 | "jeopardy", 13 | "kilt_hotpotqa", 14 | "kilt_nq", 15 | "kilt_trex", 16 | "kilt_zsre", 17 | "lama-conceptnet", 18 | "lama-google_re", 19 | "lama-squad", 20 | "lama-trex", 21 | "mc_taco", 22 | "numer_sense", 23 | "quoref", 24 | "ropes", 25 | "search_qa", 26 | "squad-no_context", 27 | "squad-with_context", 28 | "superglue-multirc", 29 | "superglue-record", 30 | "tweet_qa", 31 | "web_questions" 32 | ], 33 | "dev": [ 34 | "ai2_arc", 35 | "aqua_rat", 36 | "codah", 37 | "commonsense_qa", 38 | "cosmos_qa", 39 | "dream", 40 | "hellaswag", 41 | "math_qa", 42 | "openbookqa", 43 | "qasc", 44 | "quail", 45 | "quarel", 46 | "quartz-no_knowledge", 47 | "quartz-with_knowledge", 48 | "race-high", 49 | "race-middle", 50 | "sciq", 51 | "social_i_qa", 52 | "superglue-copa", 53 | "swag", 54 | "wino_grande", 55 | "wiqa" 56 | ], 57 | "test": [] 58 | } -------------------------------------------------------------------------------- /dataloader/custom_tasks_splits/train_non_nli_test_nli.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": [ 3 | "ade_corpus_v2-classification", 4 | "ag_news", 5 | "amazon_polarity", 6 | "circa", 7 | "climate_fever", 8 | "dbpedia_14", 9 | "discovery", 10 | "emo", 11 | "emotion", 12 | "ethos-directed_vs_generalized", 13 | "ethos-disability", 14 | "ethos-gender", 15 | "ethos-national_origin", 16 | "ethos-race", 17 | "ethos-religion", 18 | "ethos-sexual_orientation", 19 | "financial_phrasebank", 20 | "glue-cola", 21 | "glue-mrpc", 22 | "glue-qqp", 23 | "glue-sst2", 24 | "google_wellformed_query", 25 | "hate_speech18", 26 | "hate_speech_offensive", 27 | "hatexplain", 28 | "health_fact", 29 | "imdb", 30 | "kilt_fever", 31 | "liar", 32 | "medical_questions_pairs", 33 | "onestop_english", 34 | "paws", 35 | "poem_sentiment", 36 | "rotten_tomatoes", 37 | "scicite", 38 | "sick", 39 | "sms_spam", 40 | "superglue-wic", 41 | "superglue-wsc", 42 | "tab_fact", 43 | "trec", 44 | "trec-finegrained", 45 | "tweet_eval-emoji", 46 | "tweet_eval-emotion", 47 | "tweet_eval-hate", 48 | "tweet_eval-irony", 49 | "tweet_eval-offensive", 50 | "tweet_eval-sentiment", 51 | "tweet_eval-stance_abortion", 52 | "tweet_eval-stance_atheism", 53 | "tweet_eval-stance_climate", 54 | "tweet_eval-stance_feminist", 55 | "tweet_eval-stance_hillary", 56 | "wiki_auto", 57 | "wiki_qa", 58 | "yahoo_answers_topics", 59 | "yelp_polarity" 60 | ], 61 | "dev": ["anli", "glue-mnli", "glue-qnli", "glue-rte", "glue-wnli", "scitail", "sick", "superglue-cb"], 62 | "test": [] 63 | } -------------------------------------------------------------------------------- /dataloader/custom_tasks_splits/train_nonclassification_test_classification.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": ["ade_corpus_v2-dosage", "art", "biomrc", "blimp-anaphor_number_agreement", "blimp-ellipsis_n_bar_2", "blimp-sentential_negation_npi_licensor_present", "blimp-sentential_negation_npi_scope", "break-QDMR-high-level", "commonsense_qa", "crows_pairs", "dream", "duorc", "eli5-asks", "eli5-eli5", "freebase_qa", "gigaword", "hellaswag", "hotpot_qa", "kilt_ay2", "kilt_hotpotqa", "kilt_trex", "kilt_zsre", "lama-conceptnet", "lama-google_re", "lama-squad", "math_qa", "numer_sense", "openbookqa", "piqa", "proto_qa", "qa_srl", "quarel", "quartz-no_knowledge", "race-high", "reddit_tifu-title", "reddit_tifu-tldr", "ropes", "sciq", "social_i_qa", "spider", "superglue-multirc", "wiki_bio", "wikisql", "xsum", "yelp_review_full"], 3 | "dev": ["tweet_eval-stance_feminist", "ethos-national_origin", "tweet_eval-hate", "ag_news", "amazon_polarity", "hate_speech18", "poem_sentiment", "climate_fever", "medical_questions_pairs", "tweet_eval-stance_atheism"], 4 | "test": ["superglue-cb", "dbpedia_14", "wiki_qa", "emo", "yelp_polarity", "ethos-religion", "financial_phrasebank", "tab_fact", "anli", "ethos-race"] 5 | } -------------------------------------------------------------------------------- /dataloader/custom_tasks_splits/train_nonparaphrase_classification_test_paraphrase.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": ["ade_corpus_v2-classification", 3 | "ag_news", 4 | "amazon_polarity", 5 | "anli", 6 | "circa", 7 | "climate_fever", 8 | "dbpedia_14", 9 | "discovery", 10 | "emo", 11 | "emotion", 12 | "ethos-directed_vs_generalized", 13 | "ethos-disability", 14 | "ethos-gender", 15 | "ethos-national_origin", 16 | "ethos-race", 17 | "ethos-religion", 18 | "ethos-sexual_orientation", 19 | "financial_phrasebank", 20 | "glue-cola", 21 | "glue-mnli", 22 | "glue-qnli", 23 | "glue-rte", 24 | "glue-sst2", 25 | "glue-wnli", 26 | "google_wellformed_query", 27 | "hate_speech18", 28 | "hate_speech_offensive", 29 | "hatexplain", 30 | "health_fact", 31 | "imdb", 32 | "kilt_fever", 33 | "liar", 34 | "onestop_english", 35 | "poem_sentiment", 36 | "rotten_tomatoes", 37 | "scicite", 38 | "scitail", 39 | "sick", 40 | "sms_spam", 41 | "superglue-cb", 42 | "superglue-rte", 43 | "superglue-wic", 44 | "superglue-wsc", 45 | "tab_fact", 46 | "trec", 47 | "trec-finegrained", 48 | "tweet_eval-emoji", 49 | "tweet_eval-emotion", 50 | "tweet_eval-hate", 51 | "tweet_eval-irony", 52 | "tweet_eval-offensive", 53 | "tweet_eval-sentiment", 54 | "tweet_eval-stance_abortion", 55 | "tweet_eval-stance_atheism", 56 | "tweet_eval-stance_climate", 57 | "tweet_eval-stance_feminist", 58 | "tweet_eval-stance_hillary", 59 | "wiki_auto", 60 | "wiki_qa", 61 | "yahoo_answers_topics", 62 | "yelp_polarity"], 63 | "dev": [ 64 | "glue-mrpc", 65 | "glue-qqp", 66 | "medical_questions_pairs", 67 | "paws" 68 | ], 69 | "test": [] 70 | } 71 | -------------------------------------------------------------------------------- /example_scripts/collect_results.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import argparse 4 | import os 5 | 6 | # python collect_results.py --logs_dir ../models/bart-base-meta-dev-apr4 --output_file ./results_summary/bart-base-meta-dev-apr4.csv 7 | def main(): 8 | parser = argparse.ArgumentParser() 9 | 10 | parser.add_argument("--logs_dir", default=None, type=str, required=True) 11 | parser.add_argument("--output_file", default=None, type=str, required=True) 12 | 13 | args = parser.parse_args() 14 | 15 | df = pd.DataFrame(columns=["task", "entry", "dev_performance", "test_performance"]) 16 | 17 | directories = os.listdir(args.logs_dir) 18 | for directory in sorted(directories): 19 | if directory.startswith("singletask"): 20 | task = directory[11:] 21 | else: 22 | task = directory 23 | 24 | if not os.path.exists(os.path.join(args.logs_dir, directory, "result.csv")): 25 | print("Something wrong with task {}\n\n".format(task)) 26 | continue 27 | 28 | df0 = pd.read_csv(os.path.join(args.logs_dir, directory, "result.csv")) 29 | 30 | devs, tests = [], [] 31 | 32 | for idx, row in df0.iterrows(): 33 | if row["prefix"].endswith("_best"): 34 | df.loc[len(df.index)] = [task, row["prefix"][:-5], row["dev_performance"], row["test_performance"]] 35 | # print(row["prefix"], row["dev_performance"], row["test_performance"]) 36 | devs.append(row["dev_performance"]) 37 | tests.append(row["test_performance"]) 38 | 39 | if len(devs) > 0: 40 | df.loc[len(df.index)] = [task, "mean", np.mean(devs), np.mean(tests)] 41 | df.loc[len(df.index)] = [task, "std", np.std(devs), np.std(tests)] 42 | df.loc[len(df.index)] = ["", "", "", ""] 43 | 44 | df.to_csv(args.output_file) 45 | # print(directories) 46 | 47 | if __name__ == "__main__": 48 | main() -------------------------------------------------------------------------------- /example_scripts/finetune_a_list_of_tasks.sh: -------------------------------------------------------------------------------- 1 | cd .. 2 | 3 | TASKS=$1 4 | CHECKPOINT=$2 5 | IDENTIFIER=$3 6 | GPU=$4 7 | 8 | 9 | for TASK in $TASKS 10 | do 11 | 12 | echo "Task: $TASK, Checkpoint: $CHECKPOINT, Identifier: $IDENTIFIER" 13 | 14 | CUDA_VISIBLE_DEVICES=$GPU \ 15 | python tune_hps_singletask.py \ 16 | --task_dir data/${TASK}/ \ 17 | --checkpoint $CHECKPOINT \ 18 | --do_train \ 19 | --do_predict \ 20 | --learning_rate_list 1e-5 2e-5 5e-5 \ 21 | --bsz_list 2 4 8 \ 22 | --total_steps 1000 \ 23 | --eval_period 100 \ 24 | --warmup_steps 100 \ 25 | --max_grad_norm 0.1 \ 26 | --weight_decay 0.01 \ 27 | --model facebook/bart-base \ 28 | --output_dir models/${IDENTIFIER}/singletask-${TASK} \ 29 | --gradient_accumulation_steps 1 \ 30 | --predict_batch_size 32 \ 31 | --num_train_epochs 1000; 32 | 33 | done -------------------------------------------------------------------------------- /example_scripts/finetune_boolq.sh: -------------------------------------------------------------------------------- 1 | cd .. 2 | 3 | CUDA_VISIBLE_DEVICES=2 \ 4 | python cli_singletask.py \ 5 | --train_file data/boolq/boolq_16_100_train.tsv \ 6 | --dev_file data/boolq/boolq_16_100_dev.tsv \ 7 | --test_file data/boolq/boolq_16_100_test.tsv \ 8 | --do_train \ 9 | --do_predict \ 10 | --total_steps 1000 \ 11 | --eval_period 100 \ 12 | --warmup_steps 100 \ 13 | --model facebook/bart-base \ 14 | --output_dir models/singletask-boolq-no-hp \ 15 | --train_batch_size 2 \ 16 | --learning_rate 1e-5; -------------------------------------------------------------------------------- /example_scripts/finetune_boolq_tune_hp.sh: -------------------------------------------------------------------------------- 1 | cd .. 2 | 3 | python tune_hps_singletask.py \ 4 | --task_dir data/boolq/ \ 5 | --do_train \ 6 | --do_predict \ 7 | --learning_rate_list 1e-5 2e-5 5e-5 \ 8 | --bsz_list 2 4 8 \ 9 | --total_steps 1000 \ 10 | --eval_period 100 \ 11 | --warmup_steps 100 \ 12 | --model facebook/bart-base \ 13 | --output_dir models/singletask-boolq \ 14 | --predict_batch_size 32 \ -------------------------------------------------------------------------------- /example_scripts/upstream_maml.sh: -------------------------------------------------------------------------------- 1 | cd .. 2 | 3 | TASK_SPLIT=dataloader/custom_tasks_splits/random.json 4 | CUDA_VISIBLE_DEVICES=0 \ 5 | python cli_maml.py \ 6 | --do_train \ 7 | --learning_rate 1e-5 \ 8 | --output_dir models/upstream-maml \ 9 | --custom_tasks_splits ${TASK_SPLIT} \ 10 | --total_steps 6000 \ 11 | --warmup_steps 360 \ 12 | --train_batch_size 1 \ 13 | --gradient_accumulation_steps 4 \ 14 | --num_train_epochs 40; 15 | 16 | -------------------------------------------------------------------------------- /example_scripts/upstream_multitask.sh: -------------------------------------------------------------------------------- 1 | cd .. 2 | 3 | TASK_SPLIT=dataloader/custom_tasks_splits/random.json 4 | CUDA_VISIBLE_DEVICES=0 \ 5 | python cli_multitask.py \ 6 | --do_train \ 7 | --train_dir data \ 8 | --custom_tasks_splits ${TASK_SPLIT} \ 9 | --total_steps 17450 \ 10 | --warmup_steps 1047 \ 11 | --model facebook/bart-base \ 12 | --output_dir models/upstream-multitask \ 13 | --train_batch_size 32 \ 14 | --num_train_epochs 10; -------------------------------------------------------------------------------- /tasks/acronym_identification.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class AcronymIdentification(FewshotGymTextToTextDataset): 8 | 9 | # 0:"B-long", 1:"B-short", 2:"I-long", 3:"I-short", 4:"O" 10 | 11 | def __init__(self): 12 | self.hf_identifier = "acronym_identification" 13 | self.task_type = "sequence tagging" 14 | self.license = "CC BY-NC-SA 4.0" 15 | 16 | self.labels = { 17 | 0:"B-long", 18 | 1:"B-short", 19 | 2:"I-long", 20 | 3:"I-short", 21 | 4:"O", 22 | } 23 | 24 | def get_acronym_and_full_name(self, dp): 25 | labels = dp["labels"] 26 | short_st = labels.index(1) 27 | short_ed = short_st 28 | while labels[short_ed + 1] == 3: 29 | short_ed += 1 30 | acronym = " ".join(dp["tokens"][short_st: short_ed+1]) 31 | 32 | long_st = labels.index(0) 33 | long_ed = long_st 34 | while labels[long_ed + 1] == 2: 35 | long_ed += 1 36 | full_name = " ".join(dp["tokens"][long_st: long_ed+1]) 37 | 38 | return acronym, full_name 39 | 40 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 41 | lines = [] 42 | for datapoint in hf_dataset[split_name]: 43 | assert len(datapoint["labels"]) == len(datapoint["tokens"]) 44 | if datapoint["labels"].count(1) != 1 or datapoint["labels"].count(0) != 1: # only contains one acronym 45 | continue 46 | acronym, full_name = self.get_acronym_and_full_name(datapoint) 47 | lines.append((" ".join(datapoint["tokens"]) + " [SEP] acronym: " + acronym, full_name)) 48 | return lines 49 | 50 | def load_dataset(self): 51 | return datasets.load_dataset('acronym_identification') 52 | 53 | def main(): 54 | dataset = AcronymIdentification() 55 | 56 | for seed in [100, 13, 21, 42, 87]: 57 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 58 | 59 | if __name__ == "__main__": 60 | main() -------------------------------------------------------------------------------- /tasks/ade_classification.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class AdeCorpusV2_Classfication(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "ade_corpus_v2-classification" 10 | self.task_type = "classification" 11 | self.license = "unknown" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "Not Related", 16 | 1: "Related", 17 | } 18 | 19 | def get_train_test_lines(self, dataset): 20 | # only train set, manually split 20% data as test 21 | 22 | lines = self.map_hf_dataset_to_list(dataset, "train") 23 | 24 | np.random.seed(42) 25 | np.random.shuffle(lines) 26 | 27 | n = len(lines) 28 | 29 | train_lines = lines[:int(0.8*n)] 30 | test_lines = lines[int(0.8*n):] 31 | 32 | return train_lines, test_lines 33 | 34 | 35 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 36 | lines = [] 37 | for datapoint in hf_dataset[split_name]: 38 | # line[0]: input; line[1]: output 39 | lines.append((datapoint["text"], self.label[datapoint["label"]])) 40 | return lines 41 | 42 | def load_dataset(self): 43 | return datasets.load_dataset('ade_corpus_v2', 'Ade_corpus_v2_classification') 44 | 45 | def main(): 46 | dataset = AdeCorpusV2_Classfication() 47 | 48 | for seed in [100, 13, 21, 42, 87]: 49 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 50 | 51 | if __name__ == "__main__": 52 | main() -------------------------------------------------------------------------------- /tasks/ade_dosage.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class AdeCorpusV2_Dosage(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "ade_corpus_v2-dosage" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_train_test_lines(self, dataset): 15 | # only train set, manually split 20% data as test 16 | lines = self.map_hf_dataset_to_list(dataset, "train") 17 | 18 | np.random.seed(42) 19 | np.random.shuffle(lines) 20 | 21 | n = len(lines) 22 | 23 | train_lines = lines[:int(0.8*n)] 24 | test_lines = lines[int(0.8*n):] 25 | 26 | return train_lines, test_lines 27 | 28 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 29 | lines = [] 30 | for datapoint in hf_dataset[split_name]: 31 | lines.append((datapoint["text"] + " [SEP] " + datapoint["drug"], datapoint["dosage"])) 32 | return lines 33 | 34 | def load_dataset(self): 35 | return datasets.load_dataset('ade_corpus_v2', "Ade_corpus_v2_drug_dosage_relation") 36 | 37 | def main(): 38 | dataset = AdeCorpusV2_Dosage() 39 | 40 | for seed in [100, 13, 21, 42, 87]: 41 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 42 | 43 | if __name__ == "__main__": 44 | main() -------------------------------------------------------------------------------- /tasks/ade_effect.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class AdeCorpusV2_Effect(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "ade_corpus_v2-effect" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_train_test_lines(self, dataset): 15 | # only train set, manually split 20% data as test 16 | lines = self.map_hf_dataset_to_list(dataset, "train") 17 | 18 | np.random.seed(42) 19 | np.random.shuffle(lines) 20 | 21 | n = len(lines) 22 | 23 | train_lines = lines[:int(0.8*n)] 24 | test_lines = lines[int(0.8*n):] 25 | 26 | return train_lines, test_lines 27 | 28 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 29 | lines = [] 30 | for datapoint in hf_dataset[split_name]: 31 | lines.append((datapoint["text"] + " [SEP] " + datapoint["drug"], datapoint["effect"])) 32 | return lines 33 | 34 | def load_dataset(self): 35 | return datasets.load_dataset('ade_corpus_v2', "Ade_corpus_v2_drug_ade_relation") 36 | 37 | def main(): 38 | dataset = AdeCorpusV2_Effect() 39 | 40 | for seed in [100, 13, 21, 42, 87]: 41 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 42 | 43 | if __name__ == "__main__": 44 | main() -------------------------------------------------------------------------------- /tasks/adversarial_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class AdversarialQA(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "adversarialqa" 11 | self.task_type = "text to text" 12 | self.license = "cc-by-sa-4.0" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | assert len(datapoint["answers"]["text"]) == 1 18 | lines.append(("question: " + datapoint["question"] + " context: " + datapoint["context"], "\t".join(datapoint["answers"]["text"]))) 19 | return lines 20 | 21 | def load_dataset(self): 22 | return datasets.load_dataset('adversarial_qa', "adversarialQA") 23 | 24 | def main(): 25 | dataset = AdversarialQA() 26 | 27 | for seed in [100, 13, 21, 42, 87]: 28 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 29 | 30 | if __name__ == "__main__": 31 | main() -------------------------------------------------------------------------------- /tasks/aeslc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | class AESLC(FewshotGymTextToTextDataset): 9 | 10 | def __init__(self): 11 | self.hf_identifier = "aeslc" 12 | self.task_type = "text to text" 13 | self.license = "cc-by-sa-4.0" 14 | 15 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 16 | lines = [] 17 | for datapoint in hf_dataset[split_name]: 18 | lines.append(("summarize: " + clean(datapoint["email_body"]), clean(datapoint["subject_line"]))) 19 | return lines 20 | 21 | def load_dataset(self): 22 | return datasets.load_dataset("aeslc") 23 | 24 | def main(): 25 | dataset = AESLC() 26 | 27 | for seed in [100, 13, 21, 42, 87]: 28 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 29 | 30 | if __name__ == "__main__": 31 | main() -------------------------------------------------------------------------------- /tasks/agnews.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class AGNews(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "ag_news" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "World", 16 | 1: "Sports", 17 | 2: "Business", 18 | 3: "Sci/Tech", 19 | } 20 | 21 | def get_train_test_lines(self, dataset): 22 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 23 | test_lines = self.map_hf_dataset_to_list(dataset, "test") 24 | return train_lines, test_lines 25 | 26 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 27 | lines = [] 28 | for datapoint in hf_dataset[split_name]: 29 | # line[0]: input; line[1]: output 30 | lines.append((datapoint["text"], self.label[datapoint["label"]])) 31 | return lines 32 | 33 | def load_dataset(self): 34 | return datasets.load_dataset('ag_news') 35 | 36 | def main(): 37 | dataset = AGNews() 38 | 39 | for seed in [100, 13, 21, 42, 87]: 40 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 41 | 42 | if __name__ == "__main__": 43 | main() -------------------------------------------------------------------------------- /tasks/ai2_arc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class AI2_ARC(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "ai2_arc" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_choices_and_answer_string(self, datapoint): 15 | answer_index = datapoint["answerKey"] 16 | choices_string = "" 17 | for i in range(len(datapoint["choices"]["label"])): 18 | if datapoint["choices"]["label"][i] == answer_index: 19 | answer_string = datapoint["choices"]["text"][i] 20 | choices_string += " (" + datapoint["choices"]["label"][i] + ") " + datapoint["choices"]["text"][i] 21 | return choices_string, answer_string 22 | 23 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 24 | lines = [] 25 | for datapoint in hf_dataset[split_name]: 26 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 27 | lines.append((datapoint["question"] + choices_string, answer_string)) 28 | return lines 29 | 30 | def load_dataset(self): 31 | return datasets.load_dataset("ai2_arc", "ARC-Challenge") 32 | 33 | def main(): 34 | dataset = AI2_ARC() 35 | 36 | for seed in [100, 13, 21, 42, 87]: 37 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 38 | 39 | if __name__ == "__main__": 40 | main() -------------------------------------------------------------------------------- /tasks/amazon_polarity.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class AmazonPolarity(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "amazon_polarity" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "negative", 16 | 1: "positive", 17 | } 18 | 19 | def get_train_test_lines(self, dataset): 20 | # only train set, manually split 20% data as test 21 | 22 | lines = self.map_hf_dataset_to_list(dataset, "train") 23 | 24 | np.random.seed(42) 25 | np.random.shuffle(lines) 26 | 27 | n = len(lines) 28 | 29 | train_lines = lines[:10000] 30 | test_lines = lines[10000:11000] 31 | 32 | return train_lines, test_lines 33 | 34 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 35 | lines = [] 36 | if split_name == "validation": 37 | split_name = "test" # hg datasets only has train/test 38 | for datapoint in hf_dataset[split_name]: 39 | # line[0]: input; line[1]: output 40 | lines.append(("title: " + datapoint["title"] + " [SEP] content: " + datapoint["content"], self.label[datapoint["label"]])) 41 | return lines 42 | 43 | def load_dataset(self): 44 | return datasets.load_dataset('amazon_polarity') 45 | 46 | def main(): 47 | dataset = AmazonPolarity() 48 | 49 | for seed in [100, 13, 21, 42, 87]: 50 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 51 | 52 | if __name__ == "__main__": 53 | main() -------------------------------------------------------------------------------- /tasks/anli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class ANLI(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "anli" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "entailment", 16 | 1: "neutral", 17 | 2: "contradiction", 18 | } 19 | 20 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 21 | lines = [] 22 | if split_name == "validation": 23 | split_name = "dev" 24 | split_name = split_name + "_r1" 25 | for datapoint in hf_dataset[split_name]: 26 | # line[0]: input; line[1]: output 27 | lines.append(("premise: " + datapoint["premise"].replace("\n", " ") + " [SEP] hypothesis: " + datapoint["hypothesis"].replace("\n", " "), self.label[datapoint["label"]])) 28 | return lines 29 | 30 | def load_dataset(self): 31 | return datasets.load_dataset('anli') 32 | 33 | def main(): 34 | dataset = ANLI() 35 | 36 | for seed in [100, 13, 21, 42, 87]: 37 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 38 | 39 | if __name__ == "__main__": 40 | main() -------------------------------------------------------------------------------- /tasks/app_reviews.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class AppReviews(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "app_reviews" 10 | self.task_type = "regression" 11 | self.license = "unknown" 12 | 13 | def get_train_test_lines(self, dataset): 14 | # only train set, manually split 20% data as test 15 | 16 | lines = self.map_hf_dataset_to_list(dataset, "train") 17 | 18 | np.random.seed(42) 19 | np.random.shuffle(lines) 20 | 21 | n = len(lines) 22 | 23 | train_lines = lines[:int(0.8*n)] 24 | test_lines = lines[int(0.8*n):] 25 | 26 | return train_lines, test_lines 27 | 28 | 29 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 30 | lines = [] 31 | for datapoint in hf_dataset[split_name]: 32 | # line[0]: input; line[1]: output 33 | lines.append(("review: " + datapoint["review"], datapoint["star"])) 34 | return lines 35 | 36 | def load_dataset(self): 37 | return datasets.load_dataset('app_reviews') 38 | 39 | def main(): 40 | dataset = AppReviews() 41 | 42 | for seed in [100, 13, 21, 42, 87]: 43 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 44 | 45 | if __name__ == "__main__": 46 | main() -------------------------------------------------------------------------------- /tasks/aqua_rat.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class AquaRat(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "aqua_rat" 11 | self.task_type = "text to text" 12 | self.license = "apache 2.0" 13 | 14 | 15 | def get_choices_and_answer_string(self, datapoint): 16 | answer_index = datapoint["correct"] 17 | choices_string = "" 18 | for option in datapoint["options"]: 19 | if option[0] == answer_index: 20 | answer_string = option[2:] 21 | choices_string += " (" + option[0:2] + " " + option[2:] 22 | return choices_string, answer_string 23 | 24 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 25 | lines = [] 26 | for datapoint in hf_dataset[split_name]: 27 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 28 | lines.append((datapoint["question"].replace("\n", " ") + choices_string, answer_string)) 29 | return lines 30 | 31 | def load_dataset(self): 32 | return datasets.load_dataset("aqua_rat", "raw") 33 | 34 | def main(): 35 | dataset = AquaRat() 36 | 37 | for seed in [100, 13, 21, 42, 87]: 38 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 39 | 40 | if __name__ == "__main__": 41 | main() -------------------------------------------------------------------------------- /tasks/art.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class ART(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "art" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 1: "hypothesis 1", 16 | 2: "hypothesis 2", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | input_line = "observation 1: " + datapoint["observation_1"] + " [SEP] observation 2: " + datapoint["observation_2"] + " [SEP] hypothesis 1: " + datapoint["hypothesis_1"] + " [SEP] hypothesis 2: " + datapoint["hypothesis_2"] 24 | lines.append((input_line , self.label[datapoint["label"]])) 25 | return lines 26 | 27 | def load_dataset(self): 28 | return datasets.load_dataset('art') 29 | 30 | def main(): 31 | dataset = ART() 32 | 33 | for seed in [100, 13, 21, 42, 87]: 34 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 35 | 36 | if __name__ == "__main__": 37 | main() -------------------------------------------------------------------------------- /tasks/aslg_pc12.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class ASLG_PC12(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "aslg_pc12" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_train_test_lines(self, dataset): 15 | # only train set, manually split 20% data as test 16 | 17 | lines = self.map_hf_dataset_to_list(dataset, "train") 18 | 19 | np.random.seed(42) 20 | np.random.shuffle(lines) 21 | 22 | n = len(lines) 23 | 24 | train_lines = lines[:int(0.8*n)] 25 | test_lines = lines[int(0.8*n):] 26 | 27 | return train_lines, test_lines 28 | 29 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 30 | lines = [] 31 | for datapoint in hf_dataset[split_name]: 32 | lines.append((datapoint["text"].strip(), datapoint["gloss"].strip())) 33 | return lines 34 | 35 | def load_dataset(self): 36 | return datasets.load_dataset("aslg_pc12") 37 | 38 | def main(): 39 | dataset = ASLG_PC12() 40 | 41 | for seed in [100, 13, 21, 42, 87]: 42 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 43 | 44 | if __name__ == "__main__": 45 | main() -------------------------------------------------------------------------------- /tasks/biomrc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class BioMRC(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "biomrc" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_train_test_lines(self, dataset): 15 | 16 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 17 | test_lines = self.map_hf_dataset_to_list(dataset, "validation") 18 | 19 | np.random.seed(42) 20 | np.random.shuffle(test_lines) 21 | n = len(test_lines) 22 | test_lines = test_lines[:int(0.1*n)] 23 | # using 10% of test cases, otherwise it's too slow to do evaluation 24 | 25 | return train_lines, test_lines 26 | 27 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 28 | lines = [] 29 | for datapoint in hf_dataset[split_name]: 30 | lines.append(("question: " + datapoint["title"].replace("\n", " ") + " context: " + datapoint["abstract"].replace("\n", " "), datapoint["answer"])) 31 | return lines 32 | 33 | def load_dataset(self): 34 | return datasets.load_dataset('biomrc', "biomrc_large_B") 35 | 36 | def main(): 37 | dataset = BioMRC() 38 | 39 | for seed in [100, 13, 21, 42, 87]: 40 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 41 | 42 | if __name__ == "__main__": 43 | main() -------------------------------------------------------------------------------- /tasks/boolq.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class BoolQ(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "boolq" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "no", 16 | 1: "yes", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("question: " + datapoint["question"] + " [SEP] context: " + datapoint["passage"], self.label[datapoint["answer"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('boolq') 28 | 29 | def main(): 30 | dataset = BoolQ() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/break.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Break(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self, subset_identifier): 10 | self.hf_identifier = "break-" + subset_identifier 11 | self.subset_identifier = subset_identifier 12 | self.task_type = "text to text" 13 | self.license = "unknown" 14 | 15 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 16 | lines = [] 17 | for datapoint in hf_dataset[split_name]: 18 | lines.append(("question: " + datapoint["question_text"], datapoint["decomposition"])) 19 | return lines 20 | 21 | def load_dataset(self): 22 | return datasets.load_dataset('break_data', self.subset_identifier) 23 | 24 | def main(): 25 | 26 | for subset_identifier in ["QDMR", "QDMR-high-level"]: 27 | dataset = Break(subset_identifier) 28 | 29 | for seed in [100, 13, 21, 42, 87]: 30 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 31 | 32 | if __name__ == "__main__": 33 | main() -------------------------------------------------------------------------------- /tasks/circa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Circa(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "circa" 10 | self.task_type = "classification" 11 | self.license = "cc-by-4.0" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "Yes", 16 | 1: "No", 17 | 2: "In the middle, neither yes nor no", 18 | 3: "Yes, subject to some conditions", 19 | 4: "Other", 20 | } 21 | 22 | def get_train_test_lines(self, dataset): 23 | # only train set, manually split 20% data as test 24 | 25 | lines = self.map_hf_dataset_to_list(dataset, "train") 26 | 27 | np.random.seed(42) 28 | np.random.shuffle(lines) 29 | 30 | n = len(lines) 31 | 32 | train_lines = lines[:int(0.8*n)] 33 | test_lines = lines[int(0.8*n):] 34 | 35 | return train_lines, test_lines 36 | 37 | 38 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 39 | lines = [] 40 | for datapoint in hf_dataset[split_name]: 41 | # line[0]: input; line[1]: output 42 | if datapoint["goldstandard2"] == -1: 43 | continue 44 | input_text = "context: " + datapoint["context"] + " [SEP] question X: " + datapoint["question-X"] + " [SEP] answer Y: " + datapoint["answer-Y"] 45 | lines.append((input_text, self.label[datapoint["goldstandard2"]])) 46 | return lines 47 | 48 | def load_dataset(self): 49 | return datasets.load_dataset('circa') 50 | 51 | def main(): 52 | dataset = Circa() 53 | 54 | for seed in [100, 13, 21, 42, 87]: 55 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 56 | 57 | if __name__ == "__main__": 58 | main() -------------------------------------------------------------------------------- /tasks/climate_fever.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class ClimateFever(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "climate_fever" 10 | self.task_type = "classification" 11 | self.license = "unknown" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "Supports", 16 | 1: "Refutes", 17 | 2: "Not enough info", 18 | 3: "Disputed", 19 | } 20 | 21 | def get_train_test_lines(self, dataset): 22 | # for some reason it only has a test set? 23 | 24 | lines = self.map_hf_dataset_to_list(dataset, "test") 25 | 26 | np.random.seed(42) 27 | np.random.shuffle(lines) 28 | 29 | n = len(lines) 30 | 31 | train_lines = lines[:int(0.8*n)] 32 | test_lines = lines[int(0.8*n):] 33 | 34 | return train_lines, test_lines 35 | 36 | 37 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 38 | lines = [] 39 | for datapoint in hf_dataset[split_name]: 40 | # line[0]: input; line[1]: output 41 | lines.append((datapoint["claim"], self.label[datapoint["claim_label"]])) 42 | return lines 43 | 44 | def load_dataset(self): 45 | return datasets.load_dataset('climate_fever') 46 | 47 | def main(): 48 | dataset = ClimateFever() 49 | 50 | for seed in [100, 13, 21, 42, 87]: 51 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 52 | 53 | if __name__ == "__main__": 54 | main() -------------------------------------------------------------------------------- /tasks/codah.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)"} 8 | 9 | class CODAH(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "codah" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_choices_and_answer_string(self, datapoint): 17 | answer_index = datapoint["correct_answer_idx"] 18 | choices_string = "" 19 | for idx, candidate in enumerate(datapoint["candidate_answers"]): 20 | if idx == answer_index: 21 | answer_string = candidate 22 | choices_string += " " + id2alphabet[idx] + " " + candidate 23 | return choices_string, answer_string 24 | 25 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 26 | lines = [] 27 | for datapoint in hf_dataset[split_name]: 28 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 29 | lines.append((datapoint["question_propmt"] + choices_string, answer_string)) 30 | return lines 31 | 32 | def load_dataset(self): 33 | return datasets.load_dataset("codah", "fold_0") 34 | 35 | def main(): 36 | dataset = CODAH() 37 | 38 | for seed in [100, 13, 21, 42, 87]: 39 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 40 | 41 | if __name__ == "__main__": 42 | main() -------------------------------------------------------------------------------- /tasks/commongen.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class CommonGen(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "common_gen" 11 | self.task_type = "text to text" 12 | self.license = "apache-2.0" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | d = {} 17 | for datapoint in hf_dataset[split_name]: 18 | if datapoint["concept_set_idx"] not in d: 19 | d[datapoint["concept_set_idx"]] = (datapoint["concepts"], [datapoint["target"]]) 20 | else: 21 | d[datapoint["concept_set_idx"]][1].append(datapoint["target"]) 22 | for k, v in d.items(): 23 | lines.append((", ".join(v[0]), "\t".join(v[1]))) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('common_gen') 28 | 29 | def main(): 30 | dataset = CommonGen() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/commonsense_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | class CommonsenseQA(FewshotGymTextToTextDataset): 9 | 10 | def __init__(self): 11 | self.hf_identifier = "commonsense_qa" 12 | self.task_type = "text to text" 13 | self.license = "unknown" 14 | 15 | def get_choices_and_answer_string(self, datapoint): 16 | answer_index = datapoint["answerKey"] 17 | choices_string = "" 18 | for i in range(len(datapoint["choices"]["label"])): 19 | if datapoint["choices"]["label"][i] == answer_index: 20 | answer_string = datapoint["choices"]["text"][i] 21 | choices_string += " (" + datapoint["choices"]["label"][i] + ") " + datapoint["choices"]["text"][i] 22 | return choices_string, answer_string 23 | 24 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 25 | lines = [] 26 | for datapoint in hf_dataset[split_name]: 27 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 28 | lines.append((clean(datapoint["question"]) + choices_string, answer_string)) 29 | return lines 30 | 31 | def load_dataset(self): 32 | return datasets.load_dataset("commonsense_qa") 33 | 34 | def main(): 35 | dataset = CommonsenseQA() 36 | 37 | for seed in [100, 13, 21, 42, 87]: 38 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 39 | 40 | def main_more_shots(): 41 | dataset = CommonsenseQA() 42 | 43 | for shots in [64, 128, 256, 512, 1024, 2048, 4096]: 44 | for seed in [100, 13, 21, 42, 87]: 45 | train, dev, test = dataset.generate_k_shot_data(k=shots, seed=seed, path="../data_more_shots/{}_shot".format(str(shots))) 46 | 47 | if __name__ == "__main__": 48 | main() 49 | # main_more_shots() -------------------------------------------------------------------------------- /tasks/cos_e.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)", 4: "(E)"} 9 | 10 | class CoS_E(FewshotGymTextToTextDataset): 11 | 12 | def __init__(self): 13 | self.hf_identifier = "cos_e" 14 | self.task_type = "text to text" 15 | self.license = "BSD-3" 16 | 17 | def get_choices_and_answer_string(self, datapoint): 18 | answer_string = datapoint["answer"] 19 | choices_string = "" 20 | for idx, candidate in enumerate(datapoint["choices"]): 21 | choices_string += " " + id2alphabet[idx] + " " + candidate 22 | return choices_string, answer_string 23 | 24 | 25 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 26 | lines = [] 27 | for datapoint in hf_dataset[split_name]: 28 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 29 | lines.append((clean(datapoint["question"]) + choices_string, datapoint["abstractive_explanation"])) 30 | return lines 31 | 32 | def load_dataset(self): 33 | return datasets.load_dataset("cos_e", "v1.11") 34 | 35 | def main(): 36 | dataset = CoS_E() 37 | 38 | for seed in [100, 13, 21, 42, 87]: 39 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 40 | 41 | if __name__ == "__main__": 42 | main() -------------------------------------------------------------------------------- /tasks/cosmos_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)", 4: "(E)"} 8 | 9 | class CosmosQA(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "cosmos_qa" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_choices_and_answer_string(self, datapoint): 17 | answer_key = "answer" + str(datapoint["label"]) 18 | answer_string = datapoint[answer_key] 19 | choices_string = "" 20 | for idx in range(4): 21 | choices_string += " " + id2alphabet[idx] + " " + datapoint["answer" + str(idx)] 22 | return choices_string, answer_string 23 | 24 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 25 | lines = [] 26 | for datapoint in hf_dataset[split_name]: 27 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 28 | lines.append((datapoint["question"] + " [SEP] " + datapoint["context"] + " [SEP] " + choices_string, answer_string)) 29 | return lines 30 | 31 | def load_dataset(self): 32 | return datasets.load_dataset("cosmos_qa") 33 | 34 | def main(): 35 | dataset = CosmosQA() 36 | 37 | for seed in [100, 13, 21, 42, 87]: 38 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 39 | 40 | if __name__ == "__main__": 41 | main() -------------------------------------------------------------------------------- /tasks/crawl_domain.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class CrawlDomain(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "crawl_domain" 11 | self.task_type = "text to text" 12 | self.license = "mit" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append((datapoint["example"].replace(" ", "").lower(), datapoint["example"])) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset('crawl_domain') 22 | 23 | def main(): 24 | dataset = CrawlDomain() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/crows_pairs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class CrowsPairs(FewshotGymClassificationDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "crows_pairs" 11 | self.task_type = "classification" 12 | self.license = "unknown" 13 | 14 | self.label = { 15 | 0: "sentence 1", 16 | 1: "sentence 2" 17 | } 18 | 19 | def get_train_test_lines(self, dataset): 20 | # only test set, manually split 20% data as test 21 | 22 | lines = self.map_hf_dataset_to_list(dataset, "test") 23 | 24 | np.random.seed(42) 25 | np.random.shuffle(lines) 26 | 27 | n = len(lines) 28 | 29 | train_lines = lines[:int(0.8*n)] 30 | test_lines = lines[int(0.8*n):] 31 | 32 | return train_lines, test_lines 33 | 34 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 35 | lines = [] 36 | np.random.seed(42) 37 | for datapoint in hf_dataset[split_name]: 38 | if np.random.random() > 0.5: 39 | lines.append(("sentence 1: " + datapoint["sent_more"] + " [SEP] sentence 2: " + datapoint["sent_less"], self.label[0])) 40 | else: 41 | lines.append(("sentence 1: " + datapoint["sent_less"] + " [SEP] sentence 2: " + datapoint["sent_more"], self.label[1])) 42 | return lines 43 | 44 | def load_dataset(self): 45 | return datasets.load_dataset('crows_pairs') 46 | 47 | def main(): 48 | dataset = CrowsPairs() 49 | for seed in [100, 13, 21, 42, 87]: 50 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 51 | 52 | if __name__ == "__main__": 53 | main() -------------------------------------------------------------------------------- /tasks/dbpedia_14.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | from utils import clean 7 | 8 | class DBpedia14(FewshotGymClassificationDataset): 9 | def __init__(self): 10 | self.hf_identifier = "dbpedia_14" 11 | self.task_type = "classification" 12 | self.license = "cc-by-sa-3.0" 13 | 14 | # for classification tasks, specify the meaning of each label 15 | self.label = { 16 | 0:"Company", 17 | 1:"EducationalInstitution", 18 | 2:"Artist", 19 | 3:"Athlete", 20 | 4:"OfficeHolder", 21 | 5:"MeanOfTransportation", 22 | 6:"Building", 23 | 7:"NaturalPlace", 24 | 8:"Village", 25 | 9:"Animal", 26 | 10:"Plant", 27 | 11:"Album", 28 | 12:"Film", 29 | 13:"WrittenWork", 30 | } 31 | 32 | def get_train_test_lines(self, dataset): 33 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 34 | test_lines = self.map_hf_dataset_to_list(dataset, "test") 35 | 36 | np.random.seed(42) 37 | np.random.shuffle(test_lines) 38 | n = len(test_lines) 39 | test_lines = test_lines[:int(0.05*n)] 40 | # using 10% of test cases, otherwise it's too slow to do evaluation 41 | 42 | return train_lines, test_lines 43 | 44 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 45 | lines = [] 46 | for datapoint in hf_dataset[split_name]: 47 | # line[0]: input; line[1]: output 48 | lines.append((clean(datapoint["content"]), self.label[datapoint["label"]])) 49 | return lines 50 | 51 | def load_dataset(self): 52 | return datasets.load_dataset('dbpedia_14') 53 | 54 | def main(): 55 | dataset = DBpedia14() 56 | 57 | for seed in [100, 13, 21, 42, 87]: 58 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 59 | 60 | if __name__ == "__main__": 61 | main() -------------------------------------------------------------------------------- /tasks/definite_pronoun_resolution.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class DefinitePronounResolution(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "definite_pronoun_resolution" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_train_test_lines(self, dataset): 15 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 16 | test_lines = self.map_hf_dataset_to_list(dataset, "test") 17 | 18 | return train_lines, test_lines 19 | 20 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 21 | lines = [] 22 | for datapoint in hf_dataset[split_name]: 23 | input_text = datapoint["sentence"] + " [SEP] " + datapoint["pronoun"] + " [SEP] (A) " + datapoint["candidates"][0] + " (B) " + datapoint["candidates"][1] 24 | lines.append((input_text, datapoint["candidates"][datapoint["label"]])) 25 | return lines 26 | 27 | def load_dataset(self): 28 | return datasets.load_dataset('definite_pronoun_resolution') 29 | 30 | def main(): 31 | dataset = DefinitePronounResolution() 32 | 33 | for seed in [100, 13, 21, 42, 87]: 34 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 35 | 36 | if __name__ == "__main__": 37 | main() -------------------------------------------------------------------------------- /tasks/dream.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)", 4: "(E)"} 8 | 9 | class Dream(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "dream" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_choices_and_answer_string(self, datapoint): 17 | answer_string = datapoint["answer"] 18 | choices_string = "" 19 | assert len(datapoint["choice"]) == 3 20 | for idx in range(3): 21 | choices_string += " " + id2alphabet[idx] + " " + datapoint["choice"][idx] 22 | return choices_string, answer_string 23 | 24 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 25 | lines = [] 26 | for datapoint in hf_dataset[split_name]: 27 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 28 | input_text = datapoint["question"] + " [SEP] " 29 | input_text += " ".join(datapoint["dialogue"]) + " [SEP] " 30 | lines.append((input_text + choices_string, answer_string)) 31 | return lines 32 | 33 | def load_dataset(self): 34 | return datasets.load_dataset("dream") 35 | 36 | def main(): 37 | dataset = Dream() 38 | 39 | for seed in [100, 13, 21, 42, 87]: 40 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 41 | 42 | if __name__ == "__main__": 43 | main() -------------------------------------------------------------------------------- /tasks/duorc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | class DuoRC(FewshotGymTextToTextDataset): 9 | 10 | def __init__(self): 11 | self.hf_identifier = "duorc" 12 | self.task_type = "text to text" 13 | self.license = "unknown" 14 | 15 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 16 | lines = [] 17 | for datapoint in hf_dataset[split_name]: 18 | if datapoint["no_answer"] == 1: 19 | continue 20 | 21 | if datapoint["plot"].startswith("This article's plot summary may be too long or excessively detailed. Please help improve it by removing unnecessary details and making it more concise."): 22 | continue 23 | # datapoint["plot"] = datapoint["plot"].replace("This article's plot summary may be too long or excessively detailed. Please help improve it by removing unnecessary details and making it more concise.", "").strip(" ") 24 | # assert len(datapoint["answers"]) == 1 25 | input_text = "question: " + datapoint["question"] + " context: " + datapoint["plot"].replace("\n", " ") 26 | answers = [clean(item) for item in list(set(datapoint["answers"]))] 27 | lines.append((clean(input_text), "\t".join(answers))) 28 | return lines 29 | 30 | def load_dataset(self): 31 | return datasets.load_dataset('duorc', 'SelfRC') 32 | 33 | def main(): 34 | dataset = DuoRC() 35 | 36 | for seed in [100, 13, 21, 42, 87]: 37 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 38 | 39 | if __name__ == "__main__": 40 | main() -------------------------------------------------------------------------------- /tasks/e2e_nlg_cleaned.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class E2E_NLG(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "e2e_nlg_cleaned" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | input_text = datapoint["meaning_representation"] 18 | lines.append((input_text, datapoint["human_reference"])) 19 | return lines 20 | 21 | def load_dataset(self): 22 | return datasets.load_dataset('e2e_nlg_cleaned') 23 | 24 | def main(): 25 | dataset = E2E_NLG() 26 | 27 | for seed in [100, 13, 21, 42, 87]: 28 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 29 | 30 | if __name__ == "__main__": 31 | main() -------------------------------------------------------------------------------- /tasks/eli5.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | class ELI5(FewshotGymTextToTextDataset): 9 | 10 | def __init__(self, subreddit): 11 | self.hf_identifier = "eli5-" + subreddit 12 | self.subreddit = subreddit 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_train_test_lines(self, dataset): 17 | train_lines = self.map_hf_dataset_to_list(dataset, "train_{}".format(self.subreddit)) 18 | test_lines = self.map_hf_dataset_to_list(dataset, "validation_{}".format(self.subreddit)) 19 | 20 | np.random.seed(42) 21 | np.random.shuffle(test_lines) 22 | n = len(test_lines) 23 | test_lines = test_lines[:int(0.1*n)] 24 | # using 10% of test cases, otherwise it's too slow to do evaluation 25 | 26 | return train_lines, test_lines 27 | 28 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 29 | lines = [] 30 | for datapoint in hf_dataset[split_name]: 31 | input_text = datapoint["title"] + " [SEP] " + datapoint["selftext"] 32 | lines.append((clean(input_text), "\t".join(clean(item) for item in datapoint["answers"]["text"]))) 33 | return lines 34 | 35 | def load_dataset(self): 36 | return datasets.load_dataset('eli5') 37 | 38 | def main(): 39 | for subreddit in ["eli5", "asks", "askh"]: 40 | dataset = ELI5(subreddit) 41 | 42 | for seed in [100, 13, 21, 42, 87]: 43 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 44 | 45 | if __name__ == "__main__": 46 | main() -------------------------------------------------------------------------------- /tasks/emo.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Emo(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "emo" 10 | self.task_type = "classification" 11 | self.license = "unknown" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"others", 16 | 1:"happy", 17 | 2:"sad", 18 | 3:"angry", 19 | } 20 | 21 | def get_train_test_lines(self, dataset): 22 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 23 | test_lines = self.map_hf_dataset_to_list(dataset, "test") 24 | 25 | return train_lines, test_lines 26 | 27 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 28 | lines = [] 29 | for datapoint in hf_dataset[split_name]: 30 | # line[0]: input; line[1]: output 31 | lines.append((datapoint["text"].strip(), self.label[datapoint["label"]])) 32 | return lines 33 | 34 | def load_dataset(self): 35 | return datasets.load_dataset('emo') 36 | 37 | def main(): 38 | dataset = Emo() 39 | 40 | for seed in [100, 13, 21, 42, 87]: 41 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 42 | 43 | if __name__ == "__main__": 44 | main() -------------------------------------------------------------------------------- /tasks/emotion.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Emotion(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "emotion" 10 | self.task_type = "classification" 11 | self.license = "unknown" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = ["sadness", "joy", "love", "anger", "fear", "surprise"] 15 | 16 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 17 | lines = [] 18 | for datapoint in hf_dataset[split_name]: 19 | # line[0]: input; line[1]: output 20 | lines.append((datapoint["text"].strip(), self.label[datapoint["label"]])) 21 | return lines 22 | 23 | def load_dataset(self): 24 | return datasets.load_dataset('emotion') 25 | 26 | def main(): 27 | dataset = Emotion() 28 | 29 | for seed in [100, 13, 21, 42, 87]: 30 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 31 | 32 | if __name__ == "__main__": 33 | main() -------------------------------------------------------------------------------- /tasks/empathetic_dialogues.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class EmpatheticDialogues(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "empathetic_dialogues" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_train_test_lines(self, dataset): 15 | 16 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 17 | test_lines = self.map_hf_dataset_to_list(dataset, "validation") 18 | 19 | np.random.seed(42) 20 | np.random.shuffle(test_lines) 21 | n = len(test_lines) 22 | test_lines = test_lines[:int(0.2*n)] 23 | # using 20% of test cases, otherwise it's too slow to do evaluation 24 | 25 | return train_lines, test_lines 26 | 27 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 28 | lines = [] 29 | for datapoint in hf_dataset[split_name]: 30 | input_text = datapoint["prompt"].replace("_comma_", ",") + " [SEP] " + datapoint["context"] 31 | 32 | if "hit:" in datapoint["utterance"]: 33 | continue # some bad lines 34 | lines.append((input_text, datapoint["utterance"].replace("_comma_", ",").replace("\n", " ").replace("\t", " ").replace("\r", " "))) 35 | 36 | # merge same prompts 37 | d = {} 38 | for line in lines: 39 | if line[0] in d: 40 | d[line[0]].append(line[1]) 41 | else: 42 | d[line[0]] = [line[1]] 43 | 44 | lines = [] 45 | for k, v in d.items(): 46 | lines.append((k, "\t".join(v))) 47 | 48 | return lines 49 | 50 | def load_dataset(self): 51 | return datasets.load_dataset('empathetic_dialogues') 52 | 53 | def main(): 54 | dataset = EmpatheticDialogues() 55 | 56 | for seed in [100, 13, 21, 42, 87]: 57 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 58 | 59 | if __name__ == "__main__": 60 | main() -------------------------------------------------------------------------------- /tasks/ethos.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Ethos(FewshotGymClassificationDataset): 8 | def __init__(self, dimension): 9 | self.hf_identifier = "ethos-" + dimension 10 | self.dimension = dimension 11 | self.task_type = "classification" 12 | self.license = "unknown" 13 | 14 | # for classification tasks, specify the meaning of each label 15 | if dimension == "violence": 16 | self.label = { 17 | 0: "not violent", 18 | 1: "violent", 19 | } 20 | elif dimension == "directed_vs_generalized": 21 | self.label = { 22 | 0:"generalied", 23 | 1:"directed", 24 | } 25 | else: 26 | self.label = { 27 | 0:"false", 28 | 1:"true", 29 | } 30 | 31 | def get_train_test_lines(self, dataset): 32 | lines = self.map_hf_dataset_to_list(dataset, "train") 33 | 34 | np.random.seed(42) 35 | np.random.shuffle(lines) 36 | 37 | n = len(lines) 38 | 39 | train_lines = lines[:int(0.8*n)] 40 | test_lines = lines[int(0.8*n):] 41 | 42 | return train_lines, test_lines 43 | 44 | 45 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 46 | lines = [] 47 | for datapoint in hf_dataset[split_name]: 48 | # line[0]: input; line[1]: output 49 | lines.append((datapoint["text"].strip(), self.label[datapoint[self.dimension]])) 50 | return lines 51 | 52 | def load_dataset(self): 53 | return datasets.load_dataset('ethos', 'multilabel') 54 | 55 | def main(): 56 | for dimension in ["directed_vs_generalized", "disability", "gender", "national_origin", "race", "religion", "sexual_orientation"]: 57 | dataset = Ethos(dimension) 58 | for seed in [100, 13, 21, 42, 87]: 59 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 60 | 61 | if __name__ == "__main__": 62 | main() -------------------------------------------------------------------------------- /tasks/financial_phrasebank.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class FinancialPhrasebank(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "financial_phrasebank" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"negative", 16 | 1:"neutral", 17 | 2:"positive", 18 | } 19 | 20 | def get_train_test_lines(self, dataset): 21 | # only train set, manually split 20% data as test 22 | 23 | lines = self.map_hf_dataset_to_list(dataset, "train") 24 | 25 | np.random.seed(42) 26 | np.random.shuffle(lines) 27 | 28 | n = len(lines) 29 | 30 | train_lines = lines[:int(0.8*n)] 31 | test_lines = lines[int(0.8*n):] 32 | 33 | return train_lines, test_lines 34 | 35 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 36 | lines = [] 37 | for datapoint in hf_dataset[split_name]: 38 | # line[0]: input; line[1]: output 39 | lines.append((datapoint["sentence"], self.label[datapoint["label"]])) 40 | return lines 41 | 42 | def load_dataset(self): 43 | return datasets.load_dataset('financial_phrasebank', 'sentences_allagree') 44 | 45 | def main(): 46 | dataset = FinancialPhrasebank() 47 | 48 | for seed in [100, 13, 21, 42, 87]: 49 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 50 | 51 | if __name__ == "__main__": 52 | main() -------------------------------------------------------------------------------- /tasks/freebase_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class FreebaseQA(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "freebase_qa" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | if "RawQuestion" not in datapoint or "Parses" not in datapoint: 18 | continue 19 | input_text = datapoint["RawQuestion"] 20 | 21 | all_answers = [] 22 | for item in datapoint["Parses"]["Answers"]: # why the file looks so weird... 23 | for answer_name in item["AnswersName"]: 24 | for what in answer_name: 25 | all_answers.append(what) 26 | all_answers = sorted(list(set(all_answers))) 27 | 28 | output_text = datapoint["Parses"]["Answers"][0]["AnswersName"][0][0] 29 | lines.append((input_text, "\t".join(all_answers))) 30 | return lines 31 | 32 | def load_dataset(self): 33 | return datasets.load_dataset('freebase_qa') 34 | 35 | def main(): 36 | dataset = FreebaseQA() 37 | 38 | for seed in [100, 13, 21, 42, 87]: 39 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 40 | 41 | if __name__ == "__main__": 42 | main() -------------------------------------------------------------------------------- /tasks/gigaword.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Gigaword(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "gigaword" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | input_text = datapoint["document"] 18 | output_text = datapoint["summary"] 19 | lines.append(("summarize: " + input_text, output_text)) 20 | return lines 21 | 22 | def load_dataset(self): 23 | return datasets.load_dataset('gigaword') 24 | 25 | def main(): 26 | dataset = Gigaword() 27 | 28 | for seed in [100, 13, 21, 42, 87]: 29 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 30 | 31 | if __name__ == "__main__": 32 | main() -------------------------------------------------------------------------------- /tasks/glue_cola.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Glue_Cola(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "glue-cola" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "unacceptable", 16 | 1: "acceptable", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append((datapoint["sentence"], self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('glue', 'cola') 28 | 29 | def main(): 30 | dataset = Glue_Cola() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/glue_mnli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Glue_MNLI(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "glue-mnli" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "entailment", 16 | 1: "neutral", 17 | 2: "contradiction", 18 | } 19 | 20 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 21 | lines = [] 22 | if split_name == "validation": 23 | split_name = "validation_matched" 24 | for datapoint in hf_dataset[split_name]: 25 | # line[0]: input; line[1]: output 26 | lines.append(("premise: " + datapoint["premise"] + " [SEP] hypothesis: " + datapoint["hypothesis"], self.label[datapoint["label"]])) 27 | return lines 28 | 29 | def load_dataset(self): 30 | return datasets.load_dataset('glue', 'mnli') 31 | 32 | def main(): 33 | dataset = Glue_MNLI() 34 | 35 | for seed in [100, 13, 21, 42, 87]: 36 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 37 | 38 | def main_more_shots(): 39 | dataset = Glue_MNLI() 40 | 41 | for shots in [32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]: 42 | for seed in [100, 13, 21, 42, 87]: 43 | train, dev, test = dataset.generate_k_shot_data(k=shots, seed=seed, path="../data_more_shots/{}_shot".format(str(shots))) 44 | 45 | if __name__ == "__main__": 46 | main() 47 | # main_more_shots() -------------------------------------------------------------------------------- /tasks/glue_mrpc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Glue_MRPC(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "glue-mrpc" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "not_equivalent", 16 | 1: "equivalent", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("sentence 1: " + datapoint["sentence1"] + " [SEP] sentence 2: " + datapoint["sentence2"], self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('glue', 'mrpc') 28 | 29 | def main(): 30 | dataset = Glue_MRPC() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/glue_qnli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Glue_QNLI(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "glue-qnli" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "entailment", 16 | 1: "not_entailment", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("question: " + datapoint["question"] + " [SEP] sentence: " + datapoint["sentence"], self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('glue', 'qnli') 28 | 29 | def main(): 30 | dataset = Glue_QNLI() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/glue_qqp.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Glue_QQP(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "glue-qqp" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "not_duplicate", 16 | 1: "duplicate", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("question 1: " + datapoint["question1"] + " [SEP] question 2: " + datapoint["question2"], self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('glue', 'qqp') 28 | 29 | def main(): 30 | dataset = Glue_QQP() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | def main_more_shots(): 36 | dataset = Glue_QQP() 37 | 38 | for shots in [32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]: 39 | for seed in [100, 13, 21, 42, 87]: 40 | train, dev, test = dataset.generate_k_shot_data(k=shots, seed=seed, path="../data_more_shots/{}_shot".format(str(shots))) 41 | 42 | 43 | if __name__ == "__main__": 44 | main() 45 | # main_more_shots() -------------------------------------------------------------------------------- /tasks/glue_rte.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Glue_RTE(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "glue-rte" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "entailment", 16 | 1: "not_entailment", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("sentence 1: " + datapoint["sentence1"] + " [SEP] sentence 2: " + datapoint["sentence2"], self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('glue', 'rte') 28 | 29 | def main(): 30 | dataset = Glue_RTE() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/glue_sst2.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Glue_SST2(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "glue-sst2" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "negative", 16 | 1: "positive", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("sentence: " + datapoint["sentence"], self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('glue', 'sst2') 28 | 29 | def main(): 30 | dataset = Glue_SST2() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/glue_wnli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Glue_WNLI(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "glue-wnli" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"not_entailment", 16 | 1:"entailment" 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("sentence 1: " + datapoint["sentence1"] + " [SEP] sentence 2: " + datapoint["sentence2"], self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('glue', 'wnli') 28 | 29 | def main(): 30 | dataset = Glue_WNLI() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/google_wellformed_query.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class GoogleWellformedQuery(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "google_wellformed_query" 10 | self.task_type = "regression" 11 | self.license = "unknown" 12 | 13 | self.label = { 14 | 0: "not well-formed", 15 | 1: "well-formed" 16 | } 17 | 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | if datapoint["rating"] < 0.4: 23 | lines.append((datapoint["content"].strip(), self.label[0])) 24 | elif datapoint["rating"] > 0.6: 25 | lines.append((datapoint["content"].strip(), self.label[1])) 26 | return lines 27 | 28 | def load_dataset(self): 29 | return datasets.load_dataset('google_wellformed_query') 30 | 31 | def main(): 32 | dataset = GoogleWellformedQuery() 33 | for seed in [100, 13, 21, 42, 87]: 34 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 35 | 36 | if __name__ == "__main__": 37 | main() -------------------------------------------------------------------------------- /tasks/hate_speech18.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class HateSpeech18(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "hate_speech18" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "noHate", 16 | 1: "hate", 17 | } 18 | 19 | self.license = "cc-by-sa-3.0" 20 | 21 | def get_train_test_lines(self, dataset): 22 | # only train set, manually split 20% data as test 23 | 24 | lines = self.map_hf_dataset_to_list(dataset, "train") 25 | 26 | np.random.seed(42) 27 | np.random.shuffle(lines) 28 | 29 | n = len(lines) 30 | 31 | train_lines = lines[:int(0.8*n)] 32 | test_lines = lines[int(0.8*n):] 33 | 34 | return train_lines, test_lines 35 | 36 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 37 | lines = [] 38 | for datapoint in hf_dataset[split_name]: 39 | # line[0]: input; line[1]: output 40 | if datapoint["label"] > 1: # only deal with hate/nohate 41 | continue 42 | lines.append((datapoint["text"], self.label[datapoint["label"]])) 43 | return lines 44 | 45 | def load_dataset(self): 46 | return datasets.load_dataset('hate_speech18') 47 | 48 | def main(): 49 | dataset = HateSpeech18() 50 | 51 | for seed in [100, 13, 21, 42, 87]: 52 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 53 | 54 | if __name__ == "__main__": 55 | main() -------------------------------------------------------------------------------- /tasks/hate_speech_offensive.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class HateSpeechOffensive(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "hate_speech_offensive" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"hate speech", 16 | 1:"offensive language", 17 | 2:"neither", 18 | } 19 | 20 | def get_train_test_lines(self, dataset): 21 | # only train set, manually split 20% data as test 22 | 23 | lines = self.map_hf_dataset_to_list(dataset, "train") 24 | 25 | np.random.seed(42) 26 | np.random.shuffle(lines) 27 | 28 | n = len(lines) 29 | 30 | train_lines = lines[:int(0.8*n)] 31 | test_lines = lines[int(0.8*n):] 32 | 33 | return train_lines, test_lines 34 | 35 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 36 | lines = [] 37 | for datapoint in hf_dataset[split_name]: 38 | # line[0]: input; line[1]: output 39 | lines.append((datapoint["tweet"].replace("\n", " "), self.label[datapoint["class"]])) 40 | return lines 41 | 42 | def load_dataset(self): 43 | return datasets.load_dataset('hate_speech_offensive') 44 | 45 | def main(): 46 | dataset = HateSpeechOffensive() 47 | 48 | for seed in [100, 13, 21, 42, 87]: 49 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 50 | 51 | if __name__ == "__main__": 52 | main() -------------------------------------------------------------------------------- /tasks/hatexplain.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | from utils import get_majority 7 | 8 | class HatExplain(FewshotGymClassificationDataset): 9 | def __init__(self): 10 | self.hf_identifier = "hatexplain" 11 | 12 | self.task_type = "classification" 13 | 14 | # for classification tasks, specify the meaning of each label 15 | self.label = { 16 | 0:"hatespeech", 17 | 1:"normal", 18 | 2:"offensive", 19 | } 20 | 21 | self.license = "cc-by-4.0" 22 | 23 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 24 | lines = [] 25 | for datapoint in hf_dataset[split_name]: 26 | # line[0]: input; line[1]: output 27 | label = get_majority(datapoint["annotators"]["label"]) 28 | if label is not None: 29 | lines.append((" ".join(datapoint["post_tokens"]), self.label[label])) 30 | return lines 31 | 32 | def load_dataset(self): 33 | return datasets.load_dataset('hatexplain') 34 | 35 | def main(): 36 | dataset = HatExplain() 37 | 38 | for seed in [100, 13, 21, 42, 87]: 39 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 40 | 41 | if __name__ == "__main__": 42 | main() -------------------------------------------------------------------------------- /tasks/health_fact.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class HealthFact(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "health_fact" 10 | self.task_type = "classification" 11 | self.license = "mit" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"false", 16 | 1:"mixture", 17 | 2:"true", 18 | 3:"unproven", 19 | } 20 | 21 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 22 | lines = [] 23 | for datapoint in hf_dataset[split_name]: 24 | if datapoint["label"] < 0: 25 | continue 26 | lines.append((datapoint["claim"].strip().replace("\n", " ").replace("\r", " ").replace("\t", " "), self.label[datapoint["label"]])) 27 | return lines 28 | 29 | def load_dataset(self): 30 | return datasets.load_dataset('health_fact') 31 | 32 | def main(): 33 | dataset = HealthFact() 34 | for seed in [100, 13, 21, 42, 87]: 35 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 36 | 37 | if __name__ == "__main__": 38 | main() -------------------------------------------------------------------------------- /tasks/hellaswag.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)", 4: "(E)"} 8 | 9 | class HellaSwag(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "hellaswag" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_choices_and_answer_string(self, datapoint): 17 | answer_index = int(datapoint["label"]) 18 | choices_string = "" 19 | for i in range(len(datapoint["endings"])): 20 | if i == answer_index: 21 | answer_string = datapoint["endings"][i] 22 | choices_string += " " + id2alphabet[i] + " " + datapoint["endings"][i] 23 | return choices_string, answer_string 24 | 25 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 26 | lines = [] 27 | for datapoint in hf_dataset[split_name]: 28 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 29 | lines.append((datapoint["ctx"] + choices_string, answer_string)) 30 | return lines 31 | 32 | def load_dataset(self): 33 | return datasets.load_dataset("hellaswag") 34 | 35 | def main(): 36 | dataset = HellaSwag() 37 | 38 | for seed in [100, 13, 21, 42, 87]: 39 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 40 | 41 | if __name__ == "__main__": 42 | main() -------------------------------------------------------------------------------- /tasks/hotpot_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class HotpotQA(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "hotpot_qa" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_context(self, dp): 15 | counter = 1 16 | context = "" 17 | titles = dp["supporting_facts"]["title"] 18 | for sentences, title in zip(dp["context"]["sentences"], dp["context"]["title"]): 19 | if title in titles: 20 | context += "".join(sentences) + " " 21 | return context 22 | 23 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 24 | lines = [] 25 | for datapoint in hf_dataset[split_name]: 26 | context = self.get_context(datapoint) 27 | lines.append(("question: " + datapoint["question"] + " context: " + context.strip(), datapoint["answer"])) 28 | return lines 29 | 30 | def load_dataset(self): 31 | return datasets.load_dataset("hotpot_qa", "distractor") 32 | 33 | def main(): 34 | dataset = HotpotQA() 35 | 36 | for seed in [100, 13, 21, 42, 87]: 37 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 38 | 39 | if __name__ == "__main__": 40 | main() -------------------------------------------------------------------------------- /tasks/imdb.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | from utils import clean 7 | 8 | class IMDB(FewshotGymClassificationDataset): 9 | def __init__(self): 10 | self.hf_identifier = "imdb" 11 | 12 | self.task_type = "classification" 13 | 14 | # for classification tasks, specify the meaning of each label 15 | self.label = { 16 | 0: "negative", 17 | 1: "positive", 18 | } 19 | 20 | def get_train_test_lines(self, dataset): 21 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 22 | test_lines = self.map_hf_dataset_to_list(dataset, "test") 23 | 24 | return train_lines, test_lines 25 | 26 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 27 | lines = [] 28 | for datapoint in hf_dataset[split_name]: 29 | # line[0]: input; line[1]: output 30 | lines.append((clean(datapoint["text"]), self.label[datapoint["label"]])) 31 | return lines 32 | 33 | def load_dataset(self): 34 | return datasets.load_dataset('imdb') 35 | 36 | def main(): 37 | dataset = IMDB() 38 | 39 | for seed in [100, 13, 21, 42, 87]: 40 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 41 | 42 | if __name__ == "__main__": 43 | main() -------------------------------------------------------------------------------- /tasks/jeopardy.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Jeopardy(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "jeopardy" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_train_test_lines(self, dataset): 15 | # only train set, manually split 20% data as test 16 | 17 | lines = self.map_hf_dataset_to_list(dataset, "train") 18 | 19 | np.random.seed(42) 20 | np.random.shuffle(lines) 21 | 22 | n = len(lines) 23 | 24 | train_lines = lines[:int(0.8*n)] 25 | test_lines = lines[int(0.8*n):] 26 | 27 | return train_lines, test_lines 28 | 29 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 30 | lines = [] 31 | for datapoint in hf_dataset[split_name]: 32 | lines.append(("question: " + datapoint["question"].strip("'") + " [SEP] category: " + datapoint["category"], datapoint["answer"])) 33 | return lines 34 | 35 | def load_dataset(self): 36 | return datasets.load_dataset("jeopardy") 37 | 38 | def main(): 39 | dataset = Jeopardy() 40 | 41 | for seed in [100, 13, 21, 42, 87]: 42 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 43 | 44 | if __name__ == "__main__": 45 | main() -------------------------------------------------------------------------------- /tasks/kilt_ay2.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Kilt_AY2(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "kilt_ay2" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append((datapoint["input"].replace("\n", " "), "\t".join([item["answer"] for item in datapoint["output"]]))) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset('kilt_tasks','aidayago2') 22 | 23 | def main(): 24 | dataset = Kilt_AY2() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/kilt_fever.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | class Kilt_Fever(FewshotGymTextToTextDataset): 9 | 10 | def __init__(self): 11 | self.hf_identifier = "kilt_fever" 12 | self.task_type = "text to text" 13 | self.license = "unknown" 14 | 15 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 16 | lines = [] 17 | for datapoint in hf_dataset[split_name]: 18 | lines.append((clean(datapoint["input"]), "\t".join(list(set([item["answer"] for item in datapoint["output"]]))))) 19 | return lines 20 | 21 | def load_dataset(self): 22 | return datasets.load_dataset('kilt_tasks','fever') 23 | 24 | def main(): 25 | dataset = Kilt_Fever() 26 | 27 | for seed in [100, 13, 21, 42, 87]: 28 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 29 | 30 | if __name__ == "__main__": 31 | main() -------------------------------------------------------------------------------- /tasks/kilt_hotpotqa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Kilt_NQ(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "kilt_hotpotqa" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append((datapoint["input"].replace("\n", " "), "\t".join([item["answer"] for item in datapoint["output"]]))) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset('kilt_tasks','hotpotqa') 22 | 23 | def main(): 24 | dataset = Kilt_NQ() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/kilt_nq.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Kilt_NQ(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "kilt_nq" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append((datapoint["input"].replace("\n", " "), "\t".join(list(set([item["answer"] for item in datapoint["output"] if len(item["answer"])>0]))))) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset('kilt_tasks','nq') 22 | 23 | def main(): 24 | dataset = Kilt_NQ() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/kilt_trex.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Kilt_TREX(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "kilt_trex" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append((datapoint["input"].replace("\n", " "), "\t".join(list(set([item["answer"] for item in datapoint["output"] if len(item["answer"])>0]))))) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset('kilt_tasks','trex') 22 | 23 | def main(): 24 | dataset = Kilt_TREX() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/kilt_wow.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Kilt_WoW(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "kilt_wow" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append((datapoint["input"].replace("\n", " "), "\t".join([item["answer"] for item in datapoint["output"]]))) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset('kilt_tasks','wow') 22 | 23 | def main(): 24 | dataset = Kilt_WoW() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/kilt_zsre.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Kilt_ZSRE(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "kilt_zsre" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append((datapoint["input"].replace("\n", " "), "\t".join(list(set([item["answer"] for item in datapoint["output"] if len(item["answer"])>0]))))) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset('kilt_tasks','structured_zeroshot') 22 | 23 | def main(): 24 | dataset = Kilt_ZSRE() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/lama.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class LAMA(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self, subset_name): 10 | self.hf_identifier = "lama-" + subset_name 11 | self.subset_name = subset_name 12 | self.task_type = "text to text" 13 | self.license = "unknown" 14 | 15 | def get_train_test_lines(self, dataset): 16 | # only train set, manually split 20% data as test 17 | 18 | lines = self.map_hf_dataset_to_list(dataset, "train") 19 | 20 | np.random.seed(42) 21 | np.random.shuffle(lines) 22 | 23 | n = len(lines) 24 | 25 | train_lines = lines[:int(0.8*n)] 26 | if self.subset_name == "trex": # trex is too large 27 | test_lines = lines[int(0.99*n):] 28 | else: 29 | test_lines = lines[int(0.8*n):] 30 | 31 | 32 | return train_lines, test_lines 33 | 34 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 35 | lines = [] 36 | for datapoint in hf_dataset[split_name]: 37 | if self.subset_name in ["trex", "google_re"]: 38 | input_text = datapoint["template"].replace("[X]", datapoint["sub_label"]).replace("[Y]", "[MASK]") 39 | else: 40 | input_text = datapoint["masked_sentence"] 41 | lines.append((input_text, datapoint["obj_label"])) 42 | return lines 43 | 44 | def load_dataset(self): 45 | return datasets.load_dataset("lama", self.subset_name) 46 | 47 | def main(): 48 | for subset in ["trex", "squad", "google_re", "conceptnet"]: 49 | dataset = LAMA(subset) 50 | 51 | for seed in [100, 13, 21, 42, 87]: 52 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 53 | 54 | if __name__ == "__main__": 55 | main() -------------------------------------------------------------------------------- /tasks/liar.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Liar(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "liar" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"false", 16 | 1:"half-true", 17 | 2:"mostly-true", 18 | 3:"true", 19 | 4:"barely-true", 20 | 5:"pants-fire", 21 | } 22 | 23 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 24 | lines = [] 25 | for datapoint in hf_dataset[split_name]: 26 | # line[0]: input; line[1]: output 27 | input_text = "statement: " + datapoint["statement"] + " [SEP] speaker: " + datapoint["speaker"] + " [SEP] context: " + datapoint["context"] 28 | lines.append((input_text.replace("\n", " ").replace("\r", " ").replace("\t", " "), self.label[datapoint["label"]])) 29 | return lines 30 | 31 | def load_dataset(self): 32 | return datasets.load_dataset('liar') 33 | 34 | def main(): 35 | dataset = Liar() 36 | 37 | for seed in [100, 13, 21, 42, 87]: 38 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 39 | 40 | if __name__ == "__main__": 41 | main() -------------------------------------------------------------------------------- /tasks/limit.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Limit(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "limit" 11 | self.task_type = "sequence tagging" # ? 12 | self.license = "unknown" 13 | 14 | def get_train_test_lines(self, dataset): 15 | # only train set, manually split 20% data as test 16 | 17 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 18 | test_lines = self.map_hf_dataset_to_list(dataset, "test") 19 | 20 | return train_lines, test_lines 21 | 22 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 23 | lines = [] 24 | for datapoint in hf_dataset[split_name]: 25 | if datapoint["motion"] != "yes": 26 | continue 27 | entities = [item["entity"] for item in datapoint["motion_entities"]] 28 | lines.append((datapoint["sentence"].strip(), " [SEP] ".join(entities))) 29 | return lines 30 | 31 | def load_dataset(self): 32 | return datasets.load_dataset('limit') 33 | 34 | def main(): 35 | dataset = Limit() 36 | 37 | for seed in [100, 13, 21, 42, 87]: 38 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 39 | 40 | if __name__ == "__main__": 41 | main() -------------------------------------------------------------------------------- /tasks/math_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | 8 | class MathQA(FewshotGymTextToTextDataset): 9 | 10 | def __init__(self): 11 | self.hf_identifier = "math_qa" 12 | self.task_type = "text to text" 13 | self.license = "unknown" 14 | 15 | def process_line(self, dp): 16 | options = dp["options"].split(",") 17 | choices = " (A) " + options[0][4:-1] 18 | if dp["correct"] == "a": 19 | answer = options[0][4:-1] 20 | choices += " (B) " + options[1][5:-1] 21 | if dp["correct"] == "b": 22 | answer = options[1][5:-1] 23 | choices += " (C) " + options[2][5:-1] 24 | if dp["correct"] == "c": 25 | answer = options[2][5:-1] 26 | choices += " (D) " + options[3][5:-1] 27 | if dp["correct"] == "d": 28 | answer = options[3][5:-1] 29 | choices += " (E) " + options[4][5:] 30 | if dp["correct"] == "e": 31 | answer = options[4][5:] 32 | 33 | return choices, answer 34 | 35 | 36 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 37 | lines = [] 38 | for datapoint in hf_dataset[split_name]: 39 | choices, answer = self.process_line(datapoint) 40 | if answer: 41 | lines.append((datapoint["Problem"] + choices, answer)) 42 | return lines 43 | 44 | def load_dataset(self): 45 | return datasets.load_dataset('math_qa') 46 | 47 | def main(): 48 | dataset = MathQA() 49 | 50 | for seed in [100, 13, 21, 42, 87]: 51 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 52 | 53 | if __name__ == "__main__": 54 | main() -------------------------------------------------------------------------------- /tasks/mc_taco.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class MC_TACO(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "mc_taco" 10 | self.task_type = "classification" 11 | self.license = "unknown" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "no", 16 | 1: "yes", 17 | } 18 | 19 | def get_train_test_lines(self, dataset): 20 | # only train set, manually split 20% data as test 21 | 22 | lines = self.map_hf_dataset_to_list(dataset, "validation") 23 | 24 | np.random.seed(42) 25 | np.random.shuffle(lines) 26 | 27 | n = len(lines) 28 | 29 | train_lines = lines[:int(0.8*n)] 30 | test_lines = lines[int(0.8*n):] 31 | 32 | return train_lines, test_lines 33 | 34 | 35 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 36 | lines = [] 37 | for datapoint in hf_dataset[split_name]: 38 | # line[0]: input; line[1]: output 39 | input_text = datapoint["sentence"] + " [SEP] " + datapoint["question"] + " [SEP] " + datapoint["answer"] 40 | lines.append((input_text, self.label[datapoint["label"]])) 41 | return lines 42 | 43 | def load_dataset(self): 44 | return datasets.load_dataset('mc_taco') 45 | 46 | def main(): 47 | dataset = MC_TACO() 48 | 49 | for seed in [100, 13, 21, 42, 87]: 50 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 51 | 52 | if __name__ == "__main__": 53 | main() -------------------------------------------------------------------------------- /tasks/medical_questions_pairs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class MedicalQuestionPairs(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "medical_questions_pairs" 10 | self.task_type = "classification" 11 | self.license = "unknown" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "Similar", 16 | 1: "Dissimilar", 17 | } 18 | 19 | def get_train_test_lines(self, dataset): 20 | # only train set, manually split 20% data as test 21 | 22 | lines = self.map_hf_dataset_to_list(dataset, "train") 23 | 24 | np.random.seed(42) 25 | np.random.shuffle(lines) 26 | 27 | n = len(lines) 28 | 29 | train_lines = lines[:int(0.8*n)] 30 | test_lines = lines[int(0.8*n):] 31 | 32 | return train_lines, test_lines 33 | 34 | 35 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 36 | lines = [] 37 | for datapoint in hf_dataset[split_name]: 38 | # line[0]: input; line[1]: output 39 | lines.append(("question 1: " + datapoint["question_1"] + " [SEP] question 2: " + datapoint["question_2"], self.label[datapoint["label"]])) 40 | return lines 41 | 42 | def load_dataset(self): 43 | return datasets.load_dataset('medical_questions_pairs') 44 | 45 | def main(): 46 | dataset = MedicalQuestionPairs() 47 | 48 | for seed in [100, 13, 21, 42, 87]: 49 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 50 | 51 | if __name__ == "__main__": 52 | main() -------------------------------------------------------------------------------- /tasks/mocha.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Mocha(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "mocha" 10 | self.task_type = "regression" 11 | self.license = "unknown" 12 | 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | if datapoint["score"] % 1 != 0: 18 | continue 19 | # line[0]: input; line[1]: output 20 | input_text = "question: " + datapoint["question"] + " [SEP] context: " + datapoint["context"] + " [SEP] reference: " + datapoint["reference"] + " [SEP] candidate" + datapoint["candidate"] 21 | lines.append((input_text, int(datapoint["score"]))) 22 | return lines 23 | 24 | def load_dataset(self): 25 | return datasets.load_dataset('mocha') 26 | 27 | def main(): 28 | dataset = Mocha() 29 | 30 | for seed in [100, 13, 21, 42, 87]: 31 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 32 | 33 | if __name__ == "__main__": 34 | main() -------------------------------------------------------------------------------- /tasks/multi_news.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class MultiNews(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "multi_news" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | input_text = datapoint["document"] 18 | output_text = datapoint["summary"] 19 | lines.append(("summarize: " + input_text.replace("\n", " "), output_text.replace("\n", " "))) 20 | return lines 21 | 22 | def load_dataset(self): 23 | return datasets.load_dataset('multi_news') 24 | 25 | def main(): 26 | dataset = MultiNews() 27 | 28 | for seed in [100, 13, 21, 42, 87]: 29 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 30 | 31 | if __name__ == "__main__": 32 | main() -------------------------------------------------------------------------------- /tasks/not_used/0ambig_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymTextToTextDataset 6 | 7 | class AmbigQA(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "ambig_qa" 11 | self.task_type = "text to text" 12 | self.license = "cc-by-sa-3.0" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | if len(datapoint["annotations"]["qaPairs"]) != 1: 18 | continue 19 | 20 | concat_answer = "" 21 | if len(datapoint["annotations"]["qaPairs"][0]["answer"]) == 0: 22 | continue 23 | for ans in datapoint["annotations"]["qaPairs"][0]["answer"]: 24 | concat_answer += ans[0] + " [SEP] " 25 | 26 | lines.append((datapoint["question"].replace("\n", " "), concat_answer[:-7])) # remove the last [SEP] 27 | return lines 28 | 29 | def load_dataset(self): 30 | return datasets.load_dataset("ambig_qa", "light") 31 | 32 | def main(): 33 | dataset = AmbigQA() 34 | 35 | for seed in [100, 13, 21, 42, 87]: 36 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 37 | 38 | if __name__ == "__main__": 39 | main() -------------------------------------------------------------------------------- /tasks/not_used/0bc2gm_corpus.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class BC2GM_Corpus(FewshotGymTextToTextDataset): 8 | 9 | # 0:"O", 1:"B-GENE", 2:"I-GENE" 10 | 11 | def __init__(self): 12 | self.hf_identifier = "bc2gm_corpus" 13 | self.task_type = "sequence tagging" 14 | self.license = "unknown" 15 | 16 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 17 | lines = [] 18 | for datapoint in hf_dataset[split_name]: 19 | assert len(datapoint["tokens"]) == len(datapoint["ner_tags"]) 20 | datapoint["ner_tags"] = [str(item) for item in datapoint["ner_tags"]] 21 | lines.append((" ".join(datapoint["tokens"]), " ".join(datapoint["ner_tags"]))) 22 | return lines 23 | 24 | def load_dataset(self): 25 | return datasets.load_dataset('bc2gm_corpus') 26 | 27 | def main(): 28 | dataset = BC2GM_Corpus() 29 | 30 | for seed in [100, 13, 21, 42, 87]: 31 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 32 | 33 | if __name__ == "__main__": 34 | main() -------------------------------------------------------------------------------- /tasks/not_used/0cnn_dailymail.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class CNNDailyMail(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "cnn_dailymail" 11 | self.task_type = "text to text" 12 | self.license = "apache-2.0" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append(("summarize: " + datapoint["article"], datapoint["highlights"])) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset('cnn_dailymail', '3.0.0') 22 | 23 | def main(): 24 | dataset = CNNDailyMail() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/not_used/0conll2003.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class CoNLL2003(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self, subset_identifier): 10 | self.hf_identifier = "conll2003-" + subset_identifier 11 | self.subset_identifier = subset_identifier 12 | self.task_type = "sequence tagging" 13 | self.license = "unknown" 14 | 15 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 16 | lines = [] 17 | key = self.subset_identifier + "_tags" 18 | for datapoint in hf_dataset[split_name]: 19 | assert len(datapoint[key]) == len(datapoint["tokens"]) 20 | lines.append((" ".join(datapoint["tokens"]), " ".join([str(item) for item in datapoint[key]]))) 21 | return lines 22 | 23 | def load_dataset(self): 24 | return datasets.load_dataset('conll2003') 25 | 26 | def main(): 27 | for tag in ["ner", "pos", "chunk"]: 28 | dataset = CoNLL2003(tag) 29 | 30 | for seed in [100, 13, 21, 42, 87]: 31 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 32 | 33 | if __name__ == "__main__": 34 | main() -------------------------------------------------------------------------------- /tasks/not_used/0fever.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Fever(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "fever" 10 | self.task_type = "classification" 11 | self.license = "unknown" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"REFUTES", 16 | 1:"SUPPORTS", 17 | } 18 | 19 | def get_train_test_lines(self, dataset): 20 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 21 | test_lines = self.map_hf_dataset_to_list(dataset, "labelled_dev") 22 | 23 | return train_lines, test_lines 24 | 25 | 26 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 27 | lines = [] 28 | for datapoint in hf_dataset[split_name]: 29 | if datapoint["label"] == "SUPPORTS": 30 | # line[0]: input; line[1]: output 31 | lines.append((datapoint["claim"].strip(), "Supports")) 32 | elif datapoint["label"] == "REFUTES": 33 | lines.append((datapoint["claim"].strip(), "Refutes")) 34 | return lines 35 | 36 | def load_dataset(self): 37 | return datasets.load_dataset('fever', 'v1.0') 38 | 39 | def main(): 40 | dataset = Fever() 41 | for seed in [100, 13, 21, 42, 87]: 42 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 43 | 44 | if __name__ == "__main__": 45 | main() -------------------------------------------------------------------------------- /tasks/not_used/0jnlpba.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class JNLPBA(FewshotGymTextToTextDataset): 8 | 9 | # 0:"B-long", 1:"B-short", 2:"I-long", 3:"I-short", 4:"O" 10 | 11 | def __init__(self): 12 | self.hf_identifier = "jnlpba" 13 | self.task_type = "sequence tagging" 14 | self.license = "" 15 | 16 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 17 | lines = [] 18 | for datapoint in hf_dataset[split_name]: 19 | assert len(datapoint["ner_tags"]) == len(datapoint["tokens"]) 20 | lines.append((" ".join(datapoint["tokens"]), " ".join(datapoint["ner_tags"]))) 21 | return lines 22 | 23 | def load_dataset(self): 24 | return datasets.load_dataset('jnlpba') 25 | 26 | def main(): 27 | dataset = JNLPBA() 28 | 29 | for seed in [100, 13, 21, 42, 87]: 30 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 31 | 32 | if __name__ == "__main__": 33 | main() -------------------------------------------------------------------------------- /tasks/not_used/0kilt_eli5.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Kilt_ELI5(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "kilt_eli5" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append((datapoint["input"].replace("\n", " "), datapoint["output"][0]["answer"].replace("\n", " "))) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset('kilt_tasks','eli5') 22 | 23 | def main(): 24 | dataset = Kilt_ELI5() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/not_used/0kilt_triviaqa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Kilt_TriviaQA(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "kilt_trivia" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append((datapoint["input"].replace("\n", " "), datapoint["output"][0]["answer"].replace("\n", " "))) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset('kilt_tasks','triviaqa_support_only') 22 | 23 | def main(): 24 | dataset = Kilt_TriviaQA() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/not_used/0linnaeus.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Linnaeus(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "linnaeus" 11 | self.task_type = "sequence tagging" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | if len(datapoint["tokens"]) < 10: 18 | continue 19 | assert len(datapoint["tokens"]) == len(datapoint["ner_tags"]) 20 | datapoint["ner_tags"] = [str(item) for item in datapoint["ner_tags"]] 21 | lines.append((" ".join(datapoint["tokens"]), " ".join(datapoint["ner_tags"]))) 22 | return lines 23 | 24 | def load_dataset(self): 25 | return datasets.load_dataset('linnaeus') 26 | 27 | def main(): 28 | dataset = Linnaeus() 29 | 30 | for seed in [100, 13, 21, 42, 87]: 31 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 32 | 33 | if __name__ == "__main__": 34 | main() -------------------------------------------------------------------------------- /tasks/not_used/0ncbi_disease.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class NCBI_Disease(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "ncbi_disease" 11 | self.task_type = "sequence tagging" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | assert len(datapoint["ner_tags"]) == len(datapoint["tokens"]) 18 | lines.append((" ".join(datapoint["tokens"]), " ".join([str(item) for item in datapoint["ner_tags"]]))) 19 | return lines 20 | 21 | def load_dataset(self): 22 | return datasets.load_dataset('ncbi_disease') 23 | 24 | def main(): 25 | dataset = NCBI_Disease() 26 | 27 | for seed in [100, 13, 21, 42, 87]: 28 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 29 | 30 | if __name__ == "__main__": 31 | main() -------------------------------------------------------------------------------- /tasks/not_used/0nq_open.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class NQOpen(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "nq_open" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | for answer in datapoint["answer"]: 18 | lines.append((datapoint["question"], answer)) 19 | return lines 20 | 21 | def load_dataset(self): 22 | return datasets.load_dataset('nq_open') 23 | 24 | def main(): 25 | dataset = NQOpen() 26 | 27 | for seed in [100, 13, 21, 42, 87]: 28 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 29 | 30 | if __name__ == "__main__": 31 | main() -------------------------------------------------------------------------------- /tasks/not_used/0web_of_science.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class WebOfScience(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "web_of_science" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | # unknown? 15 | self.label = { 16 | } 17 | 18 | def get_train_test_lines(self, dataset): 19 | # only train set, manually split 20% data as test 20 | 21 | lines = self.map_hf_dataset_to_list(dataset, "train") 22 | 23 | np.random.seed(42) 24 | np.random.shuffle(lines) 25 | 26 | n = len(lines) 27 | 28 | train_lines = lines[:int(0.8*n)] 29 | test_lines = lines[int(0.8*n):] 30 | 31 | return train_lines, test_lines 32 | 33 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 34 | lines = [] 35 | for datapoint in hf_dataset[split_name]: 36 | # line[0]: input; line[1]: output 37 | lines.append((datapoint["text"], datapoint["label_level_1"])) 38 | return lines 39 | 40 | def load_dataset(self): 41 | return datasets.load_dataset('web_of_science', 'WOS46985') 42 | 43 | def main(): 44 | dataset = WebOfScience() 45 | 46 | for seed in [100, 13, 21, 42, 87]: 47 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 48 | 49 | if __name__ == "__main__": 50 | main() -------------------------------------------------------------------------------- /tasks/not_used/0wiki_movies.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class WikiMovies(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "wiki_movies" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append((datapoint["question"], datapoint["answer"])) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset("wiki_movies") 22 | 23 | def main(): 24 | dataset = WikiMovies() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/not_used/0wnut_17.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)"} 8 | 9 | class WNut17(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "wnut_17" 13 | self.task_type = "sequence tagging" 14 | self.license = "unknown" 15 | 16 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 17 | 18 | lines = [] 19 | for datapoint in hf_dataset[split_name]: 20 | input_text = " ".join(datapoint["tokens"]) 21 | output_text = " ".join([str(item) for item in datapoint["ner_tags"]]) 22 | lines.append((input_text, output_text)) 23 | return lines 24 | 25 | def load_dataset(self): 26 | return datasets.load_dataset("wnut_17") 27 | 28 | def main(): 29 | dataset = WNut17() 30 | 31 | for seed in [100, 13, 21, 42, 87]: 32 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 33 | 34 | if __name__ == "__main__": 35 | main() -------------------------------------------------------------------------------- /tasks/numer_sense.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class NumerSense(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "numer_sense" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_train_test_lines(self, dataset): 15 | 16 | lines = self.map_hf_dataset_to_list(dataset, "train") 17 | 18 | np.random.seed(42) 19 | np.random.shuffle(lines) 20 | 21 | n = len(lines) 22 | 23 | train_lines = lines[:int(0.8*n)] 24 | test_lines = lines[int(0.8*n):] 25 | 26 | return train_lines, test_lines 27 | 28 | 29 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 30 | lines = [] 31 | for datapoint in hf_dataset[split_name]: 32 | lines.append((datapoint["sentence"].replace("", "[MASK]"), datapoint["target"])) 33 | return lines 34 | 35 | def load_dataset(self): 36 | return datasets.load_dataset('numer_sense') 37 | 38 | def main(): 39 | dataset = NumerSense() 40 | 41 | for seed in [100, 13, 21, 42, 87]: 42 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 43 | 44 | if __name__ == "__main__": 45 | main() -------------------------------------------------------------------------------- /tasks/onestop_english.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class OneStopEnglish(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "onestop_english" 10 | self.task_type = "classification" 11 | self.license = "unknown" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"elementary", 16 | 1:"intermediate", 17 | 2:"advance", 18 | } 19 | 20 | def get_train_test_lines(self, dataset): 21 | # only train set, manually split 20% data as test 22 | 23 | lines = self.map_hf_dataset_to_list(dataset, "train") 24 | 25 | np.random.seed(42) 26 | np.random.shuffle(lines) 27 | 28 | n = len(lines) 29 | 30 | train_lines = lines[:int(0.8*n)] 31 | test_lines = lines[int(0.8*n):] 32 | 33 | return train_lines, test_lines 34 | 35 | 36 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 37 | lines = [] 38 | for datapoint in hf_dataset[split_name]: 39 | # line[0]: input; line[1]: output 40 | datapoint["text"] = datapoint["text"].replace("\n", " ") 41 | if datapoint["text"].startswith("Intermediate "): # some bug? 42 | datapoint["text"] = datapoint["text"][14:] 43 | lines.append((datapoint["text"], self.label[datapoint["label"]])) 44 | return lines 45 | 46 | def load_dataset(self): 47 | return datasets.load_dataset('onestop_english') 48 | 49 | def main(): 50 | dataset = OneStopEnglish() 51 | 52 | for seed in [100, 13, 21, 42, 87]: 53 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 54 | 55 | if __name__ == "__main__": 56 | main() -------------------------------------------------------------------------------- /tasks/openbookqa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class OpenbookQA(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "openbookqa" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_choices_and_answer_string(self, datapoint): 15 | answer_index = datapoint["answerKey"] 16 | choices_string = "" 17 | datapoint["choices"]["label"] = ["A", "B", "C", "D"] 18 | for i in range(len(datapoint["choices"]["label"])): 19 | if datapoint["choices"]["label"][i] == answer_index: 20 | answer_string = datapoint["choices"]["text"][i] 21 | choices_string += " (" + datapoint["choices"]["label"][i] + ") " + datapoint["choices"]["text"][i] 22 | return choices_string, answer_string 23 | 24 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 25 | lines = [] 26 | for datapoint in hf_dataset[split_name]: 27 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 28 | lines.append((datapoint["question_stem"] + choices_string, answer_string)) 29 | return lines 30 | 31 | def load_dataset(self): 32 | return datasets.load_dataset("openbookqa", "main") 33 | 34 | def main(): 35 | dataset = OpenbookQA() 36 | 37 | for seed in [100, 13, 21, 42, 87]: 38 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 39 | 40 | if __name__ == "__main__": 41 | main() -------------------------------------------------------------------------------- /tasks/paws.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class PAWS(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "paws" 10 | self.task_type = "classification" 11 | self.license = "unknown" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "not_duplicate", 16 | 1: "duplicate", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("sentence 1: " + datapoint["sentence1"] + " [SEP] sentence 2: " + datapoint["sentence2"], datapoint["label"])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('paws', 'labeled_final') 28 | 29 | def main(): 30 | dataset = PAWS() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/piqa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class PIQA(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "piqa" 10 | self.task_type = "classification" 11 | self.license = "unknown" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "solution 1", 16 | 1: "solution 2" 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("goal: " + datapoint["goal"] + " [SEP] solution 1" + datapoint["sol1"] + " [SEP] solution 2" + datapoint["sol2"], datapoint["label"])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('piqa') 28 | 29 | def main(): 30 | dataset = PIQA() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/poem_sentiment.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class PoemSentiment(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "poem_sentiment" 10 | self.task_type = "classification" 11 | self.license = "unknown" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"negative", 16 | 1:"positive", 17 | 2:"no_impact", 18 | 3:"mixed", 19 | } 20 | 21 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 22 | lines = [] 23 | for datapoint in hf_dataset[split_name]: 24 | # line[0]: input; line[1]: output 25 | lines.append((datapoint["verse_text"], self.label[datapoint["label"]])) 26 | return lines 27 | 28 | def load_dataset(self): 29 | return datasets.load_dataset('poem_sentiment') 30 | 31 | def main(): 32 | dataset = PoemSentiment() 33 | 34 | for seed in [100, 13, 21, 42, 87]: 35 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 36 | 37 | if __name__ == "__main__": 38 | main() -------------------------------------------------------------------------------- /tasks/proto_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | import wget 5 | import json 6 | 7 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 8 | 9 | class ProtoQA(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "proto_qa" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_train_test_lines(self, dataset): 17 | # only train set, manually split 20% data as test 18 | 19 | lines = self.map_hf_dataset_to_list(dataset, "train") 20 | 21 | np.random.seed(42) 22 | np.random.shuffle(lines) 23 | 24 | n = len(lines) 25 | 26 | train_lines = lines[:int(0.8*n)] 27 | test_lines = lines[int(0.8*n):] 28 | 29 | return train_lines, test_lines 30 | 31 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 32 | lines = [] 33 | for datapoint in hf_dataset[split_name]: 34 | lines.append((datapoint["question"]["normalized"], "\t".join(datapoint["answers"]["raw"].keys()))) 35 | return lines 36 | 37 | def load_dataset(self): 38 | if not os.path.exists("../data/proto_qa"): 39 | os.makedirs("../data/proto_qa/") 40 | if not os.path.exists("../data/proto_qa/train.jsonl"): 41 | wget.download("https://raw.githubusercontent.com/iesl/protoqa-data/master/data/train/train.jsonl", "../data/proto_qa/") 42 | with open("../data/proto_qa/train.jsonl") as fin: 43 | lines = fin.readlines() 44 | dataset = [json.loads(line) for line in lines] 45 | return {"train": dataset} 46 | 47 | def main(): 48 | dataset = ProtoQA() 49 | 50 | for seed in [100, 13, 21, 42, 87]: 51 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 52 | 53 | if __name__ == "__main__": 54 | main() -------------------------------------------------------------------------------- /tasks/qa_srl.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class QA_SRL(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "qa_srl" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append(("question: " + " ".join(datapoint["question"]) + " context: " + datapoint["sentence"], "\t".join(datapoint["answers"]))) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset('qa_srl') 22 | 23 | def main(): 24 | dataset = QA_SRL() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/qasc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class QASC(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "qasc" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_choices_and_answer_string(self, datapoint): 15 | answer_index = datapoint["answerKey"] 16 | choices_string = "" 17 | for i in range(len(datapoint["choices"]["label"])): 18 | if datapoint["choices"]["label"][i] == answer_index: 19 | answer_string = datapoint["choices"]["text"][i] 20 | choices_string += " (" + datapoint["choices"]["label"][i] + ") " + datapoint["choices"]["text"][i] 21 | return choices_string, answer_string 22 | 23 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 24 | lines = [] 25 | for datapoint in hf_dataset[split_name]: 26 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 27 | lines.append((datapoint["question"] + choices_string, answer_string)) 28 | return lines 29 | 30 | def load_dataset(self): 31 | return datasets.load_dataset("qasc") 32 | 33 | def main(): 34 | dataset = QASC() 35 | 36 | for seed in [100, 13, 21, 42, 87]: 37 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 38 | 39 | if __name__ == "__main__": 40 | main() -------------------------------------------------------------------------------- /tasks/quail.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)"} 8 | 9 | class QUAIL(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "quail" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_choices_and_answer_string(self, datapoint): 17 | answer_index = datapoint["correct_answer_id"] 18 | choices_string = "" 19 | for i, answer in enumerate(datapoint["answers"]): 20 | if i == answer_index: 21 | answer_string = datapoint["answers"][i] 22 | choices_string += id2alphabet[i] + datapoint["answers"][i] 23 | return choices_string, answer_string 24 | 25 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 26 | lines = [] 27 | for datapoint in hf_dataset[split_name]: 28 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 29 | lines.append((datapoint["question"].replace("\n", " ") + choices_string + " [SEP] " + datapoint["context"].replace("\n", " ") , answer_string)) 30 | return lines 31 | 32 | def load_dataset(self): 33 | return datasets.load_dataset("quail") 34 | 35 | def main(): 36 | dataset = QUAIL() 37 | 38 | for seed in [100, 13, 21, 42, 87]: 39 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 40 | 41 | if __name__ == "__main__": 42 | main() -------------------------------------------------------------------------------- /tasks/quarel.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class QUAREL(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "quarel" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_answer_string(self, datapoint): 15 | answer_index = datapoint["answer_index"] 16 | st1 = datapoint["question"].find("(A)") 17 | st2 = datapoint["question"].find("(B)") 18 | 19 | if answer_index == 0: 20 | answer_string = datapoint["question"][st1+4: st2] 21 | else: 22 | answer_string = datapoint["question"][st2+4: ] 23 | 24 | if answer_string.endswith("or "): 25 | answer_string = answer_string[:-3] 26 | 27 | return answer_string 28 | 29 | 30 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 31 | lines = [] 32 | for datapoint in hf_dataset[split_name]: 33 | answer_string = self.get_answer_string(datapoint) 34 | lines.append((datapoint["question"], answer_string.strip())) 35 | return lines 36 | 37 | def load_dataset(self): 38 | return datasets.load_dataset("quarel") 39 | 40 | def main(): 41 | dataset = QUAREL() 42 | 43 | for seed in [100, 13, 21, 42, 87]: 44 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 45 | 46 | if __name__ == "__main__": 47 | main() -------------------------------------------------------------------------------- /tasks/quartz.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Quartz(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self, mode): 10 | self.hf_identifier = "quartz-" + mode 11 | self.mode = mode 12 | self.task_type = "text to text" 13 | self.license = "unknown" 14 | 15 | def get_choices_and_answer_string(self, datapoint): 16 | answer_index = datapoint["answerKey"] 17 | choices_string = "" 18 | for i in range(len(datapoint["choices"]["label"])): 19 | if datapoint["choices"]["label"][i] == answer_index: 20 | answer_string = datapoint["choices"]["text"][i] 21 | choices_string += " (" + datapoint["choices"]["label"][i] + ") " + datapoint["choices"]["text"][i] 22 | return choices_string, answer_string 23 | 24 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 25 | lines = [] 26 | for datapoint in hf_dataset[split_name]: 27 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 28 | if self.mode == "with_knowledge": 29 | input_text = datapoint["question"] + datapoint["para"] + choices_string 30 | elif self.mode == "no_knowledge": 31 | input_text = datapoint["question"] + choices_string 32 | else: 33 | raise Exception() 34 | lines.append((input_text, answer_string)) 35 | return lines 36 | 37 | def load_dataset(self): 38 | return datasets.load_dataset("quartz") 39 | 40 | def main(): 41 | for mode in ["with_knowledge", "no_knowledge"]: 42 | dataset = Quartz(mode) 43 | 44 | for seed in [100, 13, 21, 42, 87]: 45 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 46 | 47 | if __name__ == "__main__": 48 | main() -------------------------------------------------------------------------------- /tasks/quoref.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Quoref(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "quoref" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | 15 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 16 | lines = [] 17 | for datapoint in hf_dataset[split_name]: 18 | input_text = "question: " + datapoint["question"] + " context: " + datapoint["context"] 19 | output_text = "\t".join(datapoint["answers"]["text"]) 20 | lines.append((input_text.replace("\n", " "), output_text)) 21 | return lines 22 | 23 | def load_dataset(self): 24 | return datasets.load_dataset("quoref") 25 | 26 | def main(): 27 | dataset = Quoref() 28 | 29 | for seed in [100, 13, 21, 42, 87]: 30 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 31 | 32 | if __name__ == "__main__": 33 | main() -------------------------------------------------------------------------------- /tasks/race.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)"} 8 | 9 | class Race(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self, subset): 12 | self.hf_identifier = "race-" + subset 13 | self.subset = subset 14 | self.task_type = "text to text" 15 | self.license = "unknown" 16 | 17 | def get_choices_and_answer_string(self, datapoint): 18 | answer_index = ord(datapoint["answer"]) - ord("A") 19 | choices_string = "" 20 | for i, ans in enumerate(datapoint["options"]): 21 | if i == answer_index: 22 | answer_string = ans.replace("\n", " ").replace("\t", " ").replace("\r", " ") 23 | choices_string += " " + id2alphabet[i] + " " + ans.replace("\n", " ").replace("\t", " ").replace("\r", " ") 24 | return choices_string, answer_string 25 | 26 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 27 | lines = [] 28 | for datapoint in hf_dataset[split_name]: 29 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 30 | input_text = datapoint["question"].replace("\n", " ").replace("\t", " ").replace("\r", " ") + choices_string + " [SEP] " + datapoint["article"].replace("\n", " ").replace("\t", " ").replace("\r", " ") 31 | lines.append((input_text, answer_string)) 32 | return lines 33 | 34 | def load_dataset(self): 35 | return datasets.load_dataset("race", self.subset) 36 | 37 | def main(): 38 | for subset in ["middle", "high"]: 39 | dataset = Race(subset) 40 | 41 | for seed in [100, 13, 21, 42, 87]: 42 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 43 | 44 | if __name__ == "__main__": 45 | main() -------------------------------------------------------------------------------- /tasks/reddit_tifu.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | class Reddit_TIFU(FewshotGymTextToTextDataset): 9 | 10 | def __init__(self, subset): 11 | self.hf_identifier = "reddit_tifu-" + subset 12 | self.subset = subset 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_train_test_lines(self, dataset): 17 | # only train set, manually split 20% data as test 18 | 19 | lines = self.map_hf_dataset_to_list(dataset, "train") 20 | 21 | np.random.seed(42) 22 | np.random.shuffle(lines) 23 | 24 | n = len(lines) 25 | 26 | train_lines = lines[:int(0.8*n)] 27 | test_lines = lines[int(0.8*n):] 28 | 29 | return train_lines, test_lines 30 | 31 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 32 | lines = [] 33 | for datapoint in hf_dataset[split_name]: 34 | lines.append(("summarize: " + clean(datapoint["documents"]), clean(datapoint[self.subset]))) 35 | return lines 36 | 37 | def load_dataset(self): 38 | return datasets.load_dataset("reddit_tifu", "long") 39 | 40 | def main(): 41 | dataset = Reddit_TIFU("tldr") 42 | 43 | for seed in [100, 13, 21, 42, 87]: 44 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 45 | 46 | dataset = Reddit_TIFU("title") 47 | 48 | for seed in [100, 13, 21, 42, 87]: 49 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 50 | 51 | if __name__ == "__main__": 52 | main() -------------------------------------------------------------------------------- /tasks/ropes.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class ROPES(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "ropes" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | input_text = "question: " + datapoint["question"] + " [SEP] situation: " + datapoint["situation"] + " [SEP] background: " + datapoint["background"] 18 | output_text = "\t".join(datapoint["answers"]["text"]) 19 | lines.append((input_text.replace("\n", " "), output_text)) 20 | return lines 21 | 22 | def load_dataset(self): 23 | return datasets.load_dataset("ropes") 24 | 25 | def main(): 26 | dataset = ROPES() 27 | 28 | for seed in [100, 13, 21, 42, 87]: 29 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 30 | 31 | def main_more_shots(): 32 | dataset = ROPES() 33 | 34 | for shots in [64, 128, 256, 512, 1024, 2048, 4096]: 35 | for seed in [100, 13, 21, 42, 87]: 36 | train, dev, test = dataset.generate_k_shot_data(k=shots, seed=seed, path="../data_more_shots/{}_shot".format(str(shots))) 37 | 38 | if __name__ == "__main__": 39 | main() 40 | # main_more_shots() -------------------------------------------------------------------------------- /tasks/rotten_tomatoes.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class RottenTomatos(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "rotten_tomatoes" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "negative", 16 | 1: "positive", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append((datapoint["text"], self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('rotten_tomatoes') 28 | 29 | def main(): 30 | dataset = RottenTomatos() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/samsum.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | class SAMSum(FewshotGymTextToTextDataset): 9 | 10 | # pip install py7zr 11 | 12 | def __init__(self): 13 | self.hf_identifier = "samsum" 14 | self.task_type = "text to text" 15 | self.license = "unknown" 16 | 17 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 18 | lines = [] 19 | for datapoint in hf_dataset[split_name]: 20 | lines.append(("summarize: " + clean(datapoint["dialogue"]), clean(datapoint["summary"]))) 21 | return lines 22 | 23 | def load_dataset(self): 24 | return datasets.load_dataset("samsum") 25 | 26 | def main(): 27 | dataset = SAMSum() 28 | 29 | for seed in [100, 13, 21, 42, 87]: 30 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 31 | 32 | if __name__ == "__main__": 33 | main() -------------------------------------------------------------------------------- /tasks/scicite.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | from utils import clean 7 | 8 | class SciCite(FewshotGymClassificationDataset): 9 | 10 | def __init__(self): 11 | self.hf_identifier = "scicite" 12 | self.task_type = "classification" 13 | self.license = "unknown" 14 | 15 | self.label = { 16 | 0:"method", 17 | 1:"background", 18 | 2:"result", 19 | } 20 | 21 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 22 | lines = [] 23 | for datapoint in hf_dataset[split_name]: 24 | input_text = datapoint["string"][datapoint["citeStart"]: datapoint["citeEnd"]] + " [SEP] " + datapoint["string"].replace("\n", " ") + " [SEP] " + datapoint["sectionName"] 25 | lines.append((clean(input_text), self.label[datapoint["label"]])) 26 | return lines 27 | 28 | def load_dataset(self): 29 | return datasets.load_dataset("scicite") 30 | 31 | def main(): 32 | dataset = SciCite() 33 | 34 | for seed in [100, 13, 21, 42, 87]: 35 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 36 | 37 | if __name__ == "__main__": 38 | main() -------------------------------------------------------------------------------- /tasks/sciq.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)"} 8 | 9 | class SciQ(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "sciq" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_choices_and_answer_string(self, datapoint): 17 | answer_string = datapoint["correct_answer"] 18 | all_answers = [datapoint["distractor1"], datapoint["distractor2"], datapoint["distractor3"], answer_string] 19 | np.random.shuffle(all_answers) 20 | 21 | choices_string = "" 22 | for i, ans in enumerate(all_answers): 23 | choices_string += " " + id2alphabet[i] + " " + ans 24 | return choices_string, answer_string 25 | 26 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 27 | np.random.seed(42) 28 | 29 | lines = [] 30 | for datapoint in hf_dataset[split_name]: 31 | if len(datapoint["support"].replace("\n", " ")) == 0: 32 | continue 33 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 34 | input_text = datapoint["question"].replace("\n", " ") + choices_string + " [SEP] " + datapoint["support"].replace("\n", " ") 35 | lines.append((input_text, answer_string)) 36 | return lines 37 | 38 | def load_dataset(self): 39 | return datasets.load_dataset("sciq") 40 | 41 | def main(): 42 | dataset = SciQ() 43 | 44 | for seed in [100, 13, 21, 42, 87]: 45 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 46 | 47 | if __name__ == "__main__": 48 | main() -------------------------------------------------------------------------------- /tasks/scitail.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class SciTail(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "scitail" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "entailment", 16 | 1: "neutral", 17 | 2: "contradiction", 18 | } 19 | 20 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 21 | lines = [] 22 | for datapoint in hf_dataset[split_name]: 23 | # line[0]: input; line[1]: output 24 | if datapoint["gold_label"] == "entailment": 25 | label = 0 26 | elif datapoint["gold_label"] == "neutral": 27 | label = 1 28 | elif datapoint["gold_label"] == "contradiction": 29 | label = 2 30 | lines.append(("sentence 1: " + datapoint["sentence1"] + " [SEP] sentence 2: " + datapoint["sentence2"], datapoint["gold_label"])) 31 | return lines 32 | 33 | def load_dataset(self): 34 | return datasets.load_dataset('scitail', 'snli_format') 35 | 36 | def main(): 37 | dataset = SciTail() 38 | 39 | for seed in [100, 13, 21, 42, 87]: 40 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 41 | 42 | if __name__ == "__main__": 43 | main() -------------------------------------------------------------------------------- /tasks/search_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class SearchQA(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "search_qa" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append(("question: " + datapoint["question"] + "category: " + datapoint["category"], datapoint["answer"])) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset("search_qa", "train_test_val") 22 | 23 | def main(): 24 | dataset = SearchQA() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/sick.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Sick(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "sick" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "entailment", 16 | 1: "neutral", 17 | 2: "contradiction", 18 | } 19 | 20 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 21 | lines = [] 22 | for datapoint in hf_dataset[split_name]: 23 | # line[0]: input; line[1]: output 24 | lines.append(("sentence 1: " + datapoint["sentence_A"] + " [SEP] sentence 2: " + datapoint["sentence_B"], self.label[datapoint["label"]])) 25 | return lines 26 | 27 | def load_dataset(self): 28 | return datasets.load_dataset('sick') 29 | 30 | def main(): 31 | dataset = Sick() 32 | 33 | for seed in [100, 13, 21, 42, 87]: 34 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 35 | 36 | if __name__ == "__main__": 37 | main() -------------------------------------------------------------------------------- /tasks/sms_spam.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class SMS_Spam(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "sms_spam" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"ham", 16 | 1:"spam", 17 | } 18 | 19 | def get_train_test_lines(self, dataset): 20 | # only train set, manually split 20% data as test 21 | 22 | lines = self.map_hf_dataset_to_list(dataset, "train") 23 | 24 | np.random.seed(42) 25 | np.random.shuffle(lines) 26 | 27 | n = len(lines) 28 | 29 | train_lines = lines[:int(0.8*n)] 30 | test_lines = lines[int(0.8*n):] 31 | 32 | return train_lines, test_lines 33 | 34 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 35 | lines = [] 36 | for datapoint in hf_dataset[split_name]: 37 | # line[0]: input; line[1]: output 38 | lines.append((datapoint["sms"].strip(), self.label[datapoint["label"]])) 39 | return lines 40 | 41 | def load_dataset(self): 42 | return datasets.load_dataset('sms_spam') 43 | 44 | def main(): 45 | dataset = SMS_Spam() 46 | 47 | for seed in [100, 13, 21, 42, 87]: 48 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 49 | 50 | if __name__ == "__main__": 51 | main() -------------------------------------------------------------------------------- /tasks/social_i_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)"} 8 | 9 | class SocialIQA(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "social_i_qa" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_choices_and_answer_string(self, datapoint): 17 | answer_idx = int(datapoint["label"]) 18 | 19 | choices_string = "" 20 | for i, ans in enumerate([datapoint["answerA"], datapoint["answerB"], datapoint["answerC"]]): 21 | if i == answer_idx-1: 22 | answer_string = ans 23 | choices_string += " " + id2alphabet[i] + " " + ans 24 | return choices_string, answer_string 25 | 26 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 27 | 28 | lines = [] 29 | for datapoint in hf_dataset[split_name]: 30 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 31 | input_text = datapoint["question"].replace("\n", " ") + choices_string + " [SEP] " + datapoint["context"].replace("\n", " ") 32 | lines.append((input_text, answer_string)) 33 | return lines 34 | 35 | def load_dataset(self): 36 | return datasets.load_dataset("social_i_qa") 37 | 38 | def main(): 39 | dataset = SocialIQA() 40 | 41 | for seed in [100, 13, 21, 42, 87]: 42 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 43 | 44 | if __name__ == "__main__": 45 | main() -------------------------------------------------------------------------------- /tasks/spider.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | class Spider(FewshotGymTextToTextDataset): 9 | 10 | def __init__(self): 11 | self.hf_identifier = "spider" 12 | self.task_type = "text to text" 13 | self.license = "unknown" 14 | 15 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 16 | lines = [] 17 | for datapoint in hf_dataset[split_name]: 18 | lines.append((clean(datapoint["question"]), clean(datapoint["query"]))) 19 | return lines 20 | 21 | def load_dataset(self): 22 | return datasets.load_dataset("spider") 23 | 24 | def main(): 25 | dataset = Spider() 26 | 27 | for seed in [100, 13, 21, 42, 87]: 28 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 29 | 30 | if __name__ == "__main__": 31 | main() -------------------------------------------------------------------------------- /tasks/squad.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | class SQuAD(FewshotGymTextToTextDataset): 9 | 10 | def __init__(self, mode): 11 | self.hf_identifier = "squad-" + mode 12 | self.mode = mode 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 17 | lines = [] 18 | for datapoint in hf_dataset[split_name]: 19 | if self.mode == "with_context": 20 | lines.append(("question: " + datapoint["question"] + " context: " + clean(datapoint["context"]), "\t".join(list(set(datapoint["answers"]["text"]))))) 21 | else: 22 | lines.append(("question: " + datapoint["question"], "\t".join(list(set(datapoint["answers"]["text"]))))) 23 | return lines 24 | 25 | def load_dataset(self): 26 | return datasets.load_dataset("squad") 27 | 28 | def main(): 29 | for mode in ["with_context", "no_context"]: 30 | dataset = SQuAD(mode) 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/superglue_cb.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Superglue_CB(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "superglue-cb" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "entailment", 16 | 1: "contradiction", 17 | 2: "neutral", 18 | } 19 | 20 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 21 | lines = [] 22 | for datapoint in hf_dataset[split_name]: 23 | # line[0]: input; line[1]: output 24 | lines.append(("premise: " + datapoint["premise"] + " [SEP] hypothesis: " + datapoint["hypothesis"], self.label[datapoint["label"]])) 25 | return lines 26 | 27 | def load_dataset(self): 28 | return datasets.load_dataset('super_glue', 'cb') 29 | 30 | def main(): 31 | dataset = Superglue_CB() 32 | 33 | for seed in [100, 13, 21, 42, 87]: 34 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 35 | 36 | if __name__ == "__main__": 37 | main() -------------------------------------------------------------------------------- /tasks/superglue_copa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Superglue_COPA(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "superglue-copa" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_choices_and_answer_string(self, datapoint): 15 | answer_index = datapoint["label"] 16 | choices_string = " (A) " + datapoint["choice1"] + " (B) " + datapoint["choice2"] 17 | if answer_index == 0: 18 | answer_string = datapoint["choice1"] 19 | else: 20 | answer_string = datapoint["choice2"] 21 | return choices_string, answer_string 22 | 23 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 24 | lines = [] 25 | for datapoint in hf_dataset[split_name]: 26 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 27 | lines.append((datapoint["premise"] + choices_string, answer_string)) 28 | return lines 29 | 30 | def load_dataset(self): 31 | return datasets.load_dataset("super_glue", "copa") 32 | 33 | def main(): 34 | dataset = Superglue_COPA() 35 | 36 | for seed in [100, 13, 21, 42, 87]: 37 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 38 | 39 | if __name__ == "__main__": 40 | main() -------------------------------------------------------------------------------- /tasks/superglue_multirc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | ID2ALPHABET = {i : "(" + chr(65+i) + ")" for i in range(26)} 8 | 9 | class Superglue_MultiRC(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "superglue-multirc" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_source_and_target_string(self, paragraph): 17 | src = "question: {}".format(paragraph[0].replace("\n", " ").replace("\r", " ").replace("\t", " ")) 18 | 19 | for idx, choice in enumerate(paragraph[2]): 20 | src += " " + ID2ALPHABET[idx] + " " + choice.replace("\n", " ").replace("\r", " ").replace("\t", " ") 21 | 22 | src += " context: {}".format(paragraph[1].replace("\n", " ").replace("\r", " ").replace("\t", " ")) 23 | 24 | correct_answers = [] 25 | for answer, label in zip(paragraph[2], paragraph[3]): 26 | if label == 1: 27 | correct_answers.append(answer) 28 | if len(correct_answers) == 0: 29 | tar = "NO ANSWER!" 30 | else: 31 | tar = " [SEP] ".join(correct_answers) 32 | 33 | return src, tar 34 | 35 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 36 | lines = [] 37 | 38 | paragraphs = {} 39 | for datapoint in hf_dataset[split_name]: 40 | if datapoint["idx"]["question"] not in paragraphs: 41 | paragraphs[datapoint["idx"]["question"]] = [datapoint["question"], datapoint["paragraph"], [datapoint["answer"]], [datapoint["label"]]] 42 | else: 43 | paragraphs[datapoint["idx"]["question"]][2].append(datapoint["answer"]) 44 | paragraphs[datapoint["idx"]["question"]][3].append(datapoint["label"]) 45 | 46 | for paragraph in paragraphs.values(): 47 | lines.append(self.get_source_and_target_string(paragraph)) 48 | return lines 49 | 50 | def load_dataset(self): 51 | return datasets.load_dataset("super_glue", "multirc") 52 | 53 | def main(): 54 | dataset = Superglue_MultiRC() 55 | 56 | for seed in [100, 13, 21, 42, 87]: 57 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 58 | 59 | if __name__ == "__main__": 60 | main() -------------------------------------------------------------------------------- /tasks/superglue_record.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Superglue_Record(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "superglue-record" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 15 | lines = [] 16 | for datapoint in hf_dataset[split_name]: 17 | lines.append(("question: " + datapoint["query"] + " context: " + datapoint["passage"].replace("\n", " "), datapoint["answers"][0])) 18 | return lines 19 | 20 | def load_dataset(self): 21 | return datasets.load_dataset("super_glue", "record") 22 | 23 | def main(): 24 | dataset = Superglue_Record() 25 | 26 | for seed in [100, 13, 21, 42, 87]: 27 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 28 | 29 | if __name__ == "__main__": 30 | main() -------------------------------------------------------------------------------- /tasks/superglue_rte.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Superglue_RTE(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "superglue-rte" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"entailment", 16 | 1:"not_entailment" 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("premise: " + datapoint["premise"].replace("\n", " ") + " [SEP] hypothesis: " + datapoint["hypothesis"].replace("\n", " "), self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('super_glue', "rte") 28 | 29 | def main(): 30 | dataset = Superglue_RTE() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/superglue_wic.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Superglue_Wic(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "superglue-wic" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "false", 16 | 1: "true", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("word: " + datapoint["word"] + " [SEP] sentence 1: " + datapoint["sentence1"] + " [SEP] sentence 2: " + datapoint["sentence2"], self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('super_glue', 'wic') 28 | 29 | def main(): 30 | dataset = Superglue_Wic() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/superglue_wsc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class Superglue_Wsc(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "superglue-wsc" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "false", 16 | 1: "true", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("text: " + datapoint["text"] + " [SEP] span1_text" + datapoint["span1_text"] + " [SEP] span2_text: " + datapoint["span2_text"], self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('super_glue', 'wsc.fixed') 28 | 29 | def main(): 30 | dataset = Superglue_Wsc() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/swag.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)"} 8 | 9 | class Swag(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "swag" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_choices_and_answer_string(self, datapoint): 17 | answer_index = datapoint["label"] 18 | candidates = [datapoint["ending0"], datapoint["ending1"], datapoint["ending2"], datapoint["ending3"]] 19 | choices_string = "" 20 | for i, ending in enumerate(candidates): 21 | if i == answer_index: 22 | answer_string = ending 23 | choices_string += " " + id2alphabet[i] + " " + ending 24 | return choices_string, answer_string 25 | 26 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 27 | lines = [] 28 | for datapoint in hf_dataset[split_name]: 29 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 30 | lines.append((datapoint["startphrase"] + choices_string, answer_string)) 31 | return lines 32 | 33 | def load_dataset(self): 34 | return datasets.load_dataset("swag", "regular") 35 | 36 | def main(): 37 | dataset = Swag() 38 | 39 | for seed in [100, 13, 21, 42, 87]: 40 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 41 | 42 | if __name__ == "__main__": 43 | main() -------------------------------------------------------------------------------- /tasks/tab_fact.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class TabFact(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "tab_fact" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "refuted", 16 | 1: "entailed", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("statement: " + datapoint["statement"] + " [SEP] table_caption: " + datapoint["table_caption"] + " [SEP] table_text: " + datapoint["table_text"].replace("\n", " [n] "), self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('tab_fact', 'tab_fact') 28 | 29 | def main(): 30 | dataset = TabFact() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/trec.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class TREC(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "trec" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"DESC", 16 | 1:"ENTY", 17 | 2:"ABBR", 18 | 3:"HUM", 19 | 4:"NUM", 20 | 5:"LOC", 21 | } 22 | 23 | def get_train_test_lines(self, dataset): 24 | # only train set, manually split 20% data as test 25 | 26 | lines = self.map_hf_dataset_to_list(dataset, "train") 27 | 28 | np.random.seed(42) 29 | np.random.shuffle(lines) 30 | 31 | n = len(lines) 32 | 33 | train_lines = lines[:int(0.8*n)] 34 | test_lines = lines[int(0.8*n):] 35 | 36 | return train_lines, test_lines 37 | 38 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 39 | lines = [] 40 | for datapoint in hf_dataset[split_name]: 41 | # line[0]: input; line[1]: output 42 | lines.append((datapoint["text"], self.label[datapoint["label-coarse"]])) 43 | return lines 44 | 45 | def load_dataset(self): 46 | return datasets.load_dataset('trec') 47 | 48 | def main(): 49 | dataset = TREC() 50 | 51 | for seed in [100, 13, 21, 42, 87]: 52 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 53 | 54 | if __name__ == "__main__": 55 | main() -------------------------------------------------------------------------------- /tasks/tweet_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | class TweetQA(FewshotGymTextToTextDataset): 9 | 10 | def __init__(self): 11 | self.hf_identifier = "tweet_qa" 12 | self.task_type = "text to text" 13 | self.license = "unknown" 14 | 15 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 16 | lines = [] 17 | for datapoint in hf_dataset[split_name]: 18 | input_text = "question: " + datapoint["Question"] + " context: " + datapoint["Tweet"] 19 | output_text = "\t".join([clean(item) for item in datapoint["Answer"]]) 20 | lines.append((clean(input_text), output_text)) 21 | return lines 22 | 23 | def load_dataset(self): 24 | return datasets.load_dataset("tweet_qa") 25 | 26 | def main(): 27 | dataset = TweetQA() 28 | 29 | for seed in [100, 13, 21, 42, 87]: 30 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 31 | 32 | if __name__ == "__main__": 33 | main() -------------------------------------------------------------------------------- /tasks/utils.py: -------------------------------------------------------------------------------- 1 | from collections import Counter 2 | 3 | def get_majority(lst): 4 | c = Counter(lst) 5 | rank = c.most_common() 6 | if len(rank) == 1: 7 | return rank[0][0] 8 | elif rank[0][1] == rank[1][1]: 9 | return None 10 | else: 11 | return rank[0][0] 12 | 13 | def clean(s): 14 | # remove special characters 15 | s = s.strip().replace("\n", " ").replace("\\n", " ").replace("\r", " ").replace("\t", " ") 16 | # remove extra whitespaces 17 | s = ' '.join(s.split()) 18 | return s 19 | -------------------------------------------------------------------------------- /tasks/web_questions.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class WebQuestions(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "web_questions" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | def get_train_test_lines(self, dataset): 15 | # only train set, manually split 20% data as test 16 | 17 | lines = self.map_hf_dataset_to_list(dataset, "train") 18 | 19 | np.random.seed(42) 20 | np.random.shuffle(lines) 21 | 22 | n = len(lines) 23 | 24 | train_lines = lines[:int(0.8*n)] 25 | test_lines = lines[int(0.8*n):] 26 | 27 | return train_lines, test_lines 28 | 29 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 30 | lines = [] 31 | np.random.seed(42) 32 | for datapoint in hf_dataset[split_name]: 33 | lines.append((datapoint["question"], "\t".join(datapoint["answers"]))) 34 | return lines 35 | 36 | def load_dataset(self): 37 | return datasets.load_dataset("web_questions") 38 | 39 | def main(): 40 | dataset = WebQuestions() 41 | 42 | for seed in [100, 13, 21, 42, 87]: 43 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 44 | 45 | if __name__ == "__main__": 46 | main() -------------------------------------------------------------------------------- /tasks/wiki_auto.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class WikiAuto(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "wiki_auto" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"notAligned", 16 | 1:"aligned", 17 | } 18 | 19 | def get_train_test_lines(self, dataset): 20 | 21 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 22 | test_lines = self.map_hf_dataset_to_list(dataset, "dev") 23 | 24 | return train_lines, test_lines 25 | 26 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 27 | lines = [] 28 | for datapoint in hf_dataset[split_name]: 29 | # line[0]: input; line[1]: output 30 | lines.append(("normal sentence: " + datapoint["normal_sentence"] + " [SEP] simple_sentence: " + datapoint["simple_sentence"], self.label[datapoint["alignment_label"]])) 31 | return lines 32 | 33 | def load_dataset(self): 34 | return datasets.load_dataset('wiki_auto', 'manual') 35 | 36 | def main(): 37 | dataset = WikiAuto() 38 | 39 | for seed in [100, 13, 21, 42, 87]: 40 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 41 | 42 | if __name__ == "__main__": 43 | main() -------------------------------------------------------------------------------- /tasks/wiki_bio.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class WikiBio(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "wiki_bio" 11 | self.task_type = "text to text" 12 | self.license = "cc-by-3.0" 13 | 14 | def get_train_test_lines(self, dataset): 15 | 16 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 17 | test_lines = self.map_hf_dataset_to_list(dataset, "val") 18 | 19 | np.random.seed(42) 20 | np.random.shuffle(test_lines) 21 | n = len(test_lines) 22 | test_lines = test_lines[:int(0.01*n)] 23 | # using 1% of test cases, otherwise it's too slow to do evaluation 24 | 25 | return train_lines, test_lines 26 | 27 | def make_input_text(self, datapoint): 28 | input_text = datapoint["input_text"]["context"].strip() + " [SEP] " 29 | if len(datapoint["input_text"]["table"]["column_header"]) != len(datapoint["input_text"]["table"]["content"]): 30 | return None 31 | for a, b in zip(datapoint["input_text"]["table"]["column_header"], datapoint["input_text"]["table"]["content"]): 32 | input_text += "{}: {} [n] ".format(a, b.strip().replace("\n", " ")) 33 | return input_text 34 | 35 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 36 | lines = [] 37 | for datapoint in hf_dataset[split_name]: 38 | input_text = self.make_input_text(datapoint) 39 | if input_text: 40 | lines.append((input_text, datapoint["target_text"].replace("\n", " "))) 41 | return lines 42 | 43 | def load_dataset(self): 44 | return datasets.load_dataset("wiki_bio") 45 | 46 | def main(): 47 | dataset = WikiBio() 48 | 49 | for seed in [100, 13, 21, 42, 87]: 50 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 51 | 52 | if __name__ == "__main__": 53 | main() -------------------------------------------------------------------------------- /tasks/wiki_qa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class WikiQA(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "wiki_qa" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0: "false", 16 | 1: "true", 17 | } 18 | 19 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 20 | lines = [] 21 | for datapoint in hf_dataset[split_name]: 22 | # line[0]: input; line[1]: output 23 | lines.append(("question: " + datapoint["question"] + " [SEP] answer: " + datapoint["answer"], self.label[datapoint["label"]])) 24 | return lines 25 | 26 | def load_dataset(self): 27 | return datasets.load_dataset('wiki_qa') 28 | 29 | def main(): 30 | dataset = WikiQA() 31 | 32 | for seed in [100, 13, 21, 42, 87]: 33 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 34 | 35 | if __name__ == "__main__": 36 | main() -------------------------------------------------------------------------------- /tasks/wiki_split.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class WikiSplit(FewshotGymTextToTextDataset): 8 | 9 | def __init__(self): 10 | self.hf_identifier = "wiki_split" 11 | self.task_type = "text to text" 12 | self.license = "unknown" 13 | 14 | 15 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 16 | lines = [] 17 | for datapoint in hf_dataset[split_name]: 18 | lines.append(("sentence 1: " + datapoint["simple_sentence_1"] + " [SEP] sentence 2: " + datapoint["simple_sentence_2"], datapoint["complex_sentence"])) 19 | return lines 20 | 21 | def load_dataset(self): 22 | return datasets.load_dataset("wiki_split") 23 | 24 | def main(): 25 | dataset = WikiSplit() 26 | 27 | for seed in [100, 13, 21, 42, 87]: 28 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 29 | 30 | if __name__ == "__main__": 31 | main() -------------------------------------------------------------------------------- /tasks/wikisql.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | from utils import clean 7 | 8 | class WikiSQL(FewshotGymTextToTextDataset): 9 | 10 | def __init__(self): 11 | self.hf_identifier = "wikisql" 12 | self.task_type = "text to text" 13 | self.license = "unknown" 14 | 15 | 16 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 17 | lines = [] 18 | for datapoint in hf_dataset[split_name]: 19 | lines.append((clean(datapoint["question"]), datapoint["sql"]["human_readable"].strip())) 20 | return lines 21 | 22 | def load_dataset(self): 23 | return datasets.load_dataset("wikisql") 24 | 25 | def main(): 26 | dataset = WikiSQL() 27 | 28 | for seed in [100, 13, 21, 42, 87]: 29 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 30 | 31 | if __name__ == "__main__": 32 | main() -------------------------------------------------------------------------------- /tasks/winogrande.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | class Winogrande(FewshotGymTextToTextDataset): 8 | def __init__(self): 9 | self.hf_identifier = "wino_grande" 10 | self.task_type = "text-to-text" 11 | 12 | 13 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 14 | lines = [] 15 | for datapoint in hf_dataset[split_name]: 16 | # line[0]: input; line[1]: output 17 | if int(datapoint["answer"]) == 1: 18 | lines.append((datapoint["sentence"] + " (A) " + datapoint["option1"] + " (B) " + datapoint["option2"], datapoint["option1"])) 19 | elif int(datapoint["answer"]) == 2: 20 | lines.append((datapoint["sentence"] + " (A) " + datapoint["option1"] + " (B) " + datapoint["option2"], datapoint["option2"])) 21 | return lines 22 | 23 | def load_dataset(self): 24 | return datasets.load_dataset('winogrande', 'winogrande_xl') 25 | 26 | def main(): 27 | dataset = Winogrande() 28 | 29 | for seed in [100, 13, 21, 42, 87]: 30 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 31 | 32 | if __name__ == "__main__": 33 | main() -------------------------------------------------------------------------------- /tasks/wiqa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)"} 8 | 9 | class WIQA(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "wiqa" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def get_choices_and_answer_string(self, datapoint): 17 | answer_idx = ord(datapoint["answer_label_as_choice"]) - ord("A") 18 | 19 | choices_string = "" 20 | for i, ans in enumerate(datapoint["choices"]["text"]): 21 | if i == answer_idx: 22 | answer_string = ans 23 | choices_string += " " + id2alphabet[i] + " " + ans 24 | return choices_string, answer_string 25 | 26 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 27 | 28 | lines = [] 29 | for datapoint in hf_dataset[split_name]: 30 | choices_string, answer_string = self.get_choices_and_answer_string(datapoint) 31 | input_text = datapoint["question_stem"].replace("\n", " ") + choices_string + " [SEP] " + " ".join(datapoint["question_para_step"]) 32 | lines.append((input_text, answer_string)) 33 | return lines 34 | 35 | def load_dataset(self): 36 | return datasets.load_dataset("wiqa") 37 | 38 | def main(): 39 | dataset = WIQA() 40 | 41 | for seed in [100, 13, 21, 42, 87]: 42 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 43 | 44 | if __name__ == "__main__": 45 | main() -------------------------------------------------------------------------------- /tasks/xsum.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymTextToTextDataset 6 | 7 | id2alphabet = {0: "(A)", 1: "(B)", 2: "(C)", 3: "(D)"} 8 | 9 | class XSum(FewshotGymTextToTextDataset): 10 | 11 | def __init__(self): 12 | self.hf_identifier = "xsum" 13 | self.task_type = "text to text" 14 | self.license = "unknown" 15 | 16 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 17 | 18 | lines = [] 19 | for datapoint in hf_dataset[split_name]: 20 | lines.append(("summarize: " + datapoint["document"].replace("\n", " "), datapoint["summary"])) 21 | return lines 22 | 23 | def load_dataset(self): 24 | return datasets.load_dataset("xsum") 25 | 26 | def main(): 27 | dataset = XSum() 28 | 29 | for seed in [100, 13, 21, 42, 87]: 30 | train, dev, test = dataset.generate_k_shot_data(k=32, seed=seed, path="../data/") 31 | 32 | if __name__ == "__main__": 33 | main() -------------------------------------------------------------------------------- /tasks/yahoo_answers_topics.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | from utils import clean 7 | 8 | class YahooAnswersTopics(FewshotGymClassificationDataset): 9 | def __init__(self): 10 | self.hf_identifier = "yahoo_answers_topics" 11 | 12 | self.task_type = "classification" 13 | 14 | # for classification tasks, specify the meaning of each label 15 | self.label = { 16 | 0:"Society & Culture", 17 | 1:"Science & Mathematics", 18 | 2:"Health", 19 | 3:"Education & Reference", 20 | 4:"Computers & Internet", 21 | 5:"Sports", 22 | 6:"Business & Finance", 23 | 7:"Entertainment & Music", 24 | 8:"Family & Relationships", 25 | 9:"Politics & Government", 26 | } 27 | 28 | def get_train_test_lines(self, dataset): 29 | # only train set, manually split 20% data as test 30 | 31 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 32 | test_lines = self.map_hf_dataset_to_list(dataset, "test") 33 | 34 | return train_lines, test_lines 35 | 36 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 37 | lines = [] 38 | for datapoint in hf_dataset[split_name]: 39 | # line[0]: input; line[1]: output 40 | input_text = "question_title: " + datapoint["question_title"] + " [SEP] question_content: " + datapoint["question_content"] + " [SEP] best_answer: " + datapoint["best_answer"] 41 | lines.append((clean(input_text), self.label[datapoint["topic"]])) 42 | return lines 43 | 44 | def load_dataset(self): 45 | return datasets.load_dataset('yahoo_answers_topics') 46 | 47 | def main(): 48 | dataset = YahooAnswersTopics() 49 | 50 | for seed in [100, 13, 21, 42, 87]: 51 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 52 | 53 | if __name__ == "__main__": 54 | main() -------------------------------------------------------------------------------- /tasks/yelp_polarity.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class YelpPolarity(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "yelp_polarity" 10 | 11 | self.task_type = "classification" 12 | 13 | # for classification tasks, specify the meaning of each label 14 | self.label = { 15 | 0:"negative", 16 | 1:"positive", 17 | } 18 | 19 | def get_train_test_lines(self, dataset): 20 | # only train set, manually split 20% data as test 21 | 22 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 23 | test_lines = self.map_hf_dataset_to_list(dataset, "test") 24 | 25 | np.random.seed(42) 26 | np.random.shuffle(test_lines) 27 | n = len(test_lines) 28 | test_lines = test_lines[:int(0.2*n)] 29 | # using 20% of test cases, otherwise it's too slow to do evaluation 30 | 31 | return train_lines, test_lines 32 | 33 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 34 | lines = [] 35 | for datapoint in hf_dataset[split_name]: 36 | lines.append((datapoint["text"].replace("\\n", " "), self.label[datapoint["label"]])) 37 | return lines 38 | 39 | def load_dataset(self): 40 | return datasets.load_dataset('yelp_polarity') 41 | 42 | def main(): 43 | dataset = YelpPolarity() 44 | 45 | for seed in [100, 13, 21, 42, 87]: 46 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 47 | 48 | if __name__ == "__main__": 49 | main() -------------------------------------------------------------------------------- /tasks/yelp_review_full.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datasets 3 | import numpy as np 4 | 5 | from fewshot_gym_dataset import FewshotGymDataset, FewshotGymClassificationDataset 6 | 7 | class YelpReviewFull(FewshotGymClassificationDataset): 8 | def __init__(self): 9 | self.hf_identifier = "yelp_review_full" 10 | 11 | self.task_type = "regression" 12 | 13 | 14 | def get_train_test_lines(self, dataset): 15 | # only train set, manually split 20% data as test 16 | 17 | train_lines = self.map_hf_dataset_to_list(dataset, "train") 18 | test_lines = self.map_hf_dataset_to_list(dataset, "test") 19 | 20 | return train_lines, test_lines 21 | 22 | def map_hf_dataset_to_list(self, hf_dataset, split_name): 23 | lines = [] 24 | for datapoint in hf_dataset[split_name]: 25 | lines.append((datapoint["text"].replace("\\n", " "), str(datapoint["label"] + 1))) 26 | return lines 27 | 28 | def load_dataset(self): 29 | return datasets.load_dataset('yelp_review_full') 30 | 31 | def main(): 32 | dataset = YelpReviewFull() 33 | 34 | for seed in [100, 13, 21, 42, 87]: 35 | train, dev, test = dataset.generate_k_shot_data(k=16, seed=seed, path="../data/") 36 | 37 | if __name__ == "__main__": 38 | main() -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | # from dataloader.default_split import DEFAULT_SPLIT 3 | import json 4 | 5 | def label_smoothed_nll_loss(lprobs, target, epsilon=0.1, ignore_index=-100): 6 | """From fairseq""" 7 | if target.dim() == lprobs.dim() - 1: 8 | target = target.unsqueeze(-1) 9 | nll_loss = -lprobs.gather(dim=-1, index=target) 10 | smooth_loss = -lprobs.sum(dim=-1, keepdim=True) 11 | if ignore_index is not None: 12 | pad_mask = target.eq(ignore_index) 13 | nll_loss.masked_fill_(pad_mask, 0.0) 14 | smooth_loss.masked_fill_(pad_mask, 0.0) 15 | else: 16 | nll_loss = nll_loss.squeeze(-1) 17 | smooth_loss = smooth_loss.squeeze(-1) 18 | 19 | nll_loss = nll_loss.sum() 20 | smooth_loss = smooth_loss.sum() 21 | eps_i = epsilon / lprobs.size(-1) 22 | loss = (1.0 - epsilon) * nll_loss + eps_i * smooth_loss 23 | return loss, nll_loss 24 | 25 | def freeze_params(model: nn.Module): 26 | """Set requires_grad=False for each of model.parameters()""" 27 | for par in model.parameters(): 28 | par.requires_grad = False 29 | 30 | 31 | def freeze_embeds(model): 32 | """Freeze token embeddings and positional embeddings for bart, just token embeddings for t5.""" 33 | model_type = model.config.model_type 34 | 35 | if model_type == "t5": 36 | freeze_params(model.shared) 37 | for d in [model.encoder, model.decoder]: 38 | freeze_params(d.embed_tokens) 39 | elif model_type == "fsmt": 40 | for d in [model.model.encoder, model.model.decoder]: 41 | freeze_params(d.embed_positions) 42 | freeze_params(d.embed_tokens) 43 | else: 44 | freeze_params(model.model.shared) 45 | for d in [model.model.encoder, model.model.decoder]: 46 | freeze_params(d.embed_positions) 47 | freeze_params(d.embed_tokens) 48 | 49 | def trim_batch( 50 | input_ids, 51 | pad_token_id, 52 | attention_mask=None, 53 | ): 54 | """Remove columns that are populated exclusively by pad_token_id""" 55 | keep_column_mask = input_ids.ne(pad_token_id).any(dim=0) 56 | if attention_mask is None: 57 | return input_ids[:, keep_column_mask] 58 | else: 59 | return (input_ids[:, keep_column_mask], attention_mask[:, keep_column_mask]) 60 | 61 | def get_tasks_list(filename, split_name): 62 | with open(filename, "r") as fin: 63 | split_dict = json.load(fin) 64 | return split_dict[split_name] --------------------------------------------------------------------------------