├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .github └── workflows │ └── cpu_ci.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .vscode └── settings.json ├── LICENSE.md ├── MANIFEST.in ├── README.md ├── elk ├── __init__.py ├── __main__.py ├── debug_logging.py ├── evaluation │ ├── __init__.py │ └── evaluate.py ├── extraction │ ├── __init__.py │ ├── balanced_sampler.py │ ├── dataset_name.py │ ├── extraction.py │ ├── generator.py │ ├── inference_server.py │ └── prompt_loading.py ├── files.py ├── metrics │ ├── __init__.py │ ├── accuracy.py │ ├── calibration.py │ ├── eval.py │ └── roc_auc.py ├── plotting │ ├── command.py │ └── visualize.py ├── promptsource │ ├── LICENSE │ ├── __init__.py │ ├── templates.py │ └── templates │ │ ├── Zaid │ │ ├── coqa_expanded │ │ │ └── templates.yaml │ │ └── quac_expanded │ │ │ └── templates.yaml │ │ ├── _default │ │ └── templates.yaml │ │ ├── _no_suffix │ │ └── templates.yaml │ │ ├── acronym_identification │ │ └── templates.yaml │ │ ├── ade_corpus_v2 │ │ ├── Ade_corpus_v2_classification │ │ │ └── templates.yaml │ │ ├── Ade_corpus_v2_drug_ade_relation │ │ │ └── templates.yaml │ │ └── Ade_corpus_v2_drug_dosage_relation │ │ │ └── templates.yaml │ │ ├── adversarial_qa │ │ ├── adversarialQA │ │ │ └── templates.yaml │ │ ├── dbert │ │ │ └── templates.yaml │ │ ├── dbidaf │ │ │ └── templates.yaml │ │ └── droberta │ │ │ └── templates.yaml │ │ ├── aeslc │ │ └── templates.yaml │ │ ├── ag_news │ │ └── templates.yaml │ │ ├── ai2_arc │ │ ├── ARC-Challenge │ │ │ └── templates.yaml │ │ └── ARC-Easy │ │ │ └── templates.yaml │ │ ├── amazon_polarity │ │ └── templates.yaml │ │ ├── amazon_reviews_multi │ │ └── en │ │ │ └── templates.yaml │ │ ├── amazon_us_reviews │ │ └── Wireless_v1_00 │ │ │ └── templates.yaml │ │ ├── ambig_qa │ │ └── light │ │ │ └── templates.yaml │ │ ├── anli │ │ └── templates.yaml │ │ ├── app_reviews │ │ └── templates.yaml │ │ ├── aqua_rat │ │ └── raw │ │ │ └── templates.yaml │ │ ├── art │ │ └── templates.yaml │ │ ├── asnq │ │ └── templates.yaml │ │ ├── asset │ │ ├── ratings │ │ │ └── templates.yaml │ │ └── simplification │ │ │ └── templates.yaml │ │ ├── banking77 │ │ └── templates.yaml │ │ ├── billsum │ │ └── templates.yaml │ │ ├── bing_coronavirus_query_set │ │ └── templates.yaml │ │ ├── biosses │ │ └── templates.yaml │ │ ├── blbooksgenre │ │ └── title_genre_classifiction │ │ │ └── templates.yaml │ │ ├── blended_skill_talk │ │ └── templates.yaml │ │ ├── boolq_pt │ │ └── templates.yaml │ │ ├── cbt │ │ ├── CN │ │ │ └── templates.yaml │ │ ├── NE │ │ │ └── templates.yaml │ │ ├── P │ │ │ └── templates.yaml │ │ ├── V │ │ │ └── templates.yaml │ │ └── raw │ │ │ └── templates.yaml │ │ ├── cc_news │ │ └── templates.yaml │ │ ├── christykoh │ │ ├── ag_news_pt │ │ │ └── templates.yaml │ │ ├── boolq_pt │ │ │ └── templates.yaml │ │ ├── imdb_ar │ │ │ └── templates.yaml │ │ ├── imdb_es │ │ │ └── templates.yaml │ │ ├── imdb_fr │ │ │ └── templates.yaml │ │ ├── imdb_pt │ │ │ └── templates.yaml │ │ └── imdb_zh │ │ │ └── templates.yaml │ │ ├── circa │ │ └── templates.yaml │ │ ├── climate_fever │ │ └── templates.yaml │ │ ├── cnn_dailymail │ │ └── 3.0.0 │ │ │ └── templates.yaml │ │ ├── codah │ │ ├── codah │ │ │ └── templates.yaml │ │ ├── fold_0 │ │ │ └── templates.yaml │ │ ├── fold_1 │ │ │ └── templates.yaml │ │ ├── fold_2 │ │ │ └── templates.yaml │ │ ├── fold_3 │ │ │ └── templates.yaml │ │ └── fold_4 │ │ │ └── templates.yaml │ │ ├── code_x_glue_tc_text_to_code │ │ └── templates.yaml │ │ ├── common_gen │ │ └── templates.yaml │ │ ├── commonsense_qa │ │ └── templates.yaml │ │ ├── conv_ai │ │ └── templates.yaml │ │ ├── conv_ai_2 │ │ └── templates.yaml │ │ ├── conv_ai_3 │ │ └── templates.yaml │ │ ├── coqa │ │ └── templates.yaml │ │ ├── cord19 │ │ └── metadata │ │ │ └── templates.yaml │ │ ├── cos_e │ │ ├── v1.0 │ │ │ └── templates.yaml │ │ └── v1.11 │ │ │ └── templates.yaml │ │ ├── cosmos_qa │ │ └── templates.yaml │ │ ├── covid_qa_castorini │ │ └── templates.yaml │ │ ├── craffel │ │ └── openai_lambada │ │ │ └── templates.yaml │ │ ├── craigslist_bargains │ │ └── templates.yaml │ │ ├── crows_pairs │ │ └── templates.yaml │ │ ├── dbpedia_14 │ │ └── templates.yaml │ │ ├── discofuse │ │ ├── discofuse-sport │ │ │ └── templates.yaml │ │ └── discofuse-wikipedia │ │ │ └── templates.yaml │ │ ├── discovery │ │ └── discovery │ │ │ └── templates.yaml │ │ ├── docred │ │ └── templates.yaml │ │ ├── dream │ │ └── templates.yaml │ │ ├── drop │ │ └── templates.yaml │ │ ├── duorc │ │ ├── ParaphraseRC │ │ │ └── templates.yaml │ │ └── SelfRC │ │ │ └── templates.yaml │ │ ├── e2e_nlg_cleaned │ │ └── templates.yaml │ │ ├── ecthr_cases │ │ └── alleged-violation-prediction │ │ │ └── templates.yaml │ │ ├── emo │ │ └── templates.yaml │ │ ├── emotion │ │ └── templates.yaml │ │ ├── enriched_web_nlg │ │ └── en │ │ │ └── templates.yaml │ │ ├── esnli │ │ └── templates.yaml │ │ ├── evidence_infer_treatment │ │ ├── 1.1 │ │ │ └── templates.yaml │ │ └── 2.0 │ │ │ └── templates.yaml │ │ ├── fever │ │ ├── v1.0 │ │ │ └── templates.yaml │ │ └── v2.0 │ │ │ └── templates.yaml │ │ ├── financial_phrasebank │ │ └── sentences_allagree │ │ │ └── templates.yaml │ │ ├── freebase_qa │ │ └── templates.yaml │ │ ├── generated_reviews_enth │ │ └── templates.yaml │ │ ├── gigaword │ │ └── templates.yaml │ │ ├── glue │ │ ├── ax │ │ │ └── templates.yaml │ │ ├── cola │ │ │ └── templates.yaml │ │ ├── mnli │ │ │ └── templates.yaml │ │ ├── mnli_matched │ │ │ └── templates.yaml │ │ ├── mnli_mismatched │ │ │ └── templates.yaml │ │ ├── mrpc │ │ │ └── templates.yaml │ │ ├── qnli │ │ │ └── templates.yaml │ │ ├── qqp │ │ │ └── templates.yaml │ │ ├── rte │ │ │ └── templates.yaml │ │ ├── sst2 │ │ │ └── templates.yaml │ │ ├── stsb │ │ │ └── templates.yaml │ │ └── wnli │ │ │ └── templates.yaml │ │ ├── google_wellformed_query │ │ └── templates.yaml │ │ ├── great_code │ │ └── templates.yaml │ │ ├── guardian_authorship │ │ ├── cross_genre_1 │ │ │ └── templates.yaml │ │ ├── cross_topic_1 │ │ │ └── templates.yaml │ │ ├── cross_topic_4 │ │ │ └── templates.yaml │ │ └── cross_topic_7 │ │ │ └── templates.yaml │ │ ├── gutenberg_time │ │ └── templates.yaml │ │ ├── hans │ │ └── templates.yaml │ │ ├── hate_speech18 │ │ └── templates.yaml │ │ ├── head_qa │ │ └── en │ │ │ └── templates.yaml │ │ ├── health_fact │ │ └── templates.yaml │ │ ├── hellaswag │ │ └── templates.yaml │ │ ├── hlgd │ │ └── templates.yaml │ │ ├── hotpot_qa │ │ ├── distractor │ │ │ └── templates.yaml │ │ └── fullwiki │ │ │ └── templates.yaml │ │ ├── humicroedit │ │ ├── subtask-1 │ │ │ └── templates.yaml │ │ └── subtask-2 │ │ │ └── templates.yaml │ │ ├── hyperpartisan_news_detection │ │ ├── byarticle │ │ │ └── templates.yaml │ │ └── bypublisher │ │ │ └── templates.yaml │ │ ├── imdb │ │ └── templates.yaml │ │ ├── jfleg │ │ └── templates.yaml │ │ ├── jigsaw_unintended_bias │ │ └── templates.yaml │ │ ├── kelm │ │ └── templates.yaml │ │ ├── kilt_tasks │ │ ├── hotpotqa │ │ │ └── templates.yaml │ │ └── nq │ │ │ └── templates.yaml │ │ ├── lama │ │ └── trex │ │ │ └── templates.yaml │ │ ├── lambada │ │ └── templates.yaml │ │ ├── lauritowal │ │ └── redefine_math │ │ │ └── templates.yaml │ │ ├── liar │ │ └── templates.yaml │ │ ├── limit │ │ └── templates.yaml │ │ ├── math_dataset │ │ ├── algebra__linear_1d │ │ │ └── templates.yaml │ │ ├── algebra__linear_1d_composed │ │ │ └── templates.yaml │ │ ├── algebra__linear_2d │ │ │ └── templates.yaml │ │ └── algebra__linear_2d_composed │ │ │ └── templates.yaml │ │ ├── math_qa │ │ └── templates.yaml │ │ ├── mc_taco │ │ └── templates.yaml │ │ ├── mdd │ │ ├── task1_qa │ │ │ └── templates.yaml │ │ ├── task2_recs │ │ │ └── templates.yaml │ │ └── task3_qarecs │ │ │ └── templates.yaml │ │ ├── medal │ │ └── templates.yaml │ │ ├── medical_questions_pairs │ │ └── templates.yaml │ │ ├── meta_woz │ │ └── dialogues │ │ │ └── templates.yaml │ │ ├── mocha │ │ └── templates.yaml │ │ ├── movie_rationales │ │ └── templates.yaml │ │ ├── multi_news │ │ └── templates.yaml │ │ ├── multi_nli │ │ └── templates.yaml │ │ ├── multi_x_science_sum │ │ └── templates.yaml │ │ ├── mwsc │ │ └── templates.yaml │ │ ├── narrativeqa │ │ └── templates.yaml │ │ ├── ncbi_disease │ │ └── templates.yaml │ │ ├── neural_code_search │ │ └── evaluation_dataset │ │ │ └── templates.yaml │ │ ├── newspop │ │ └── templates.yaml │ │ ├── nlu_evaluation_data │ │ └── templates.yaml │ │ ├── nq_open │ │ └── templates.yaml │ │ ├── numer_sense │ │ └── templates.yaml │ │ ├── onestop_english │ │ └── templates.yaml │ │ ├── openai_humaneval │ │ └── templates.yaml │ │ ├── openbookqa │ │ ├── additional │ │ │ └── templates.yaml │ │ └── main │ │ │ └── templates.yaml │ │ ├── paws-x │ │ └── en │ │ │ └── templates.yaml │ │ ├── paws │ │ ├── labeled_final │ │ │ └── templates.yaml │ │ ├── labeled_swap │ │ │ └── templates.yaml │ │ └── unlabeled_final │ │ │ └── templates.yaml │ │ ├── piqa │ │ └── templates.yaml │ │ ├── poem_sentiment │ │ └── templates.yaml │ │ ├── pubmed_qa │ │ └── pqa_labeled │ │ │ └── templates.yaml │ │ ├── qa_srl │ │ └── templates.yaml │ │ ├── qa_zre │ │ └── templates.yaml │ │ ├── qasc │ │ └── templates.yaml │ │ ├── qed │ │ └── templates.yaml │ │ ├── quac │ │ └── templates.yaml │ │ ├── quail │ │ └── templates.yaml │ │ ├── quarel │ │ └── templates.yaml │ │ ├── quartz │ │ └── templates.yaml │ │ ├── quora │ │ └── templates.yaml │ │ ├── quoref │ │ └── templates.yaml │ │ ├── race │ │ ├── all │ │ │ └── templates.yaml │ │ ├── high │ │ │ └── templates.yaml │ │ └── middle │ │ │ └── templates.yaml │ │ ├── reaganjlee │ │ ├── boolq_ar │ │ │ └── templates.yaml │ │ ├── boolq_es │ │ │ └── templates.yaml │ │ ├── boolq_fr │ │ │ └── templates.yaml │ │ ├── boolq_pt │ │ │ └── templates.yaml │ │ ├── boolq_zh │ │ │ └── templates.yaml │ │ ├── truthful_qa_mc │ │ │ └── templates.yaml │ │ ├── truthful_qa_mc_ar │ │ │ └── templates.yaml │ │ ├── truthful_qa_mc_es │ │ │ └── templates.yaml │ │ ├── truthful_qa_mc_fr │ │ │ └── templates.yaml │ │ ├── truthful_qa_mc_pt │ │ │ └── templates.yaml │ │ └── truthful_qa_mc_zh │ │ │ └── templates.yaml │ │ ├── riddle_sense │ │ └── templates.yaml │ │ ├── ropes │ │ └── templates.yaml │ │ ├── rotten_tomatoes │ │ └── templates.yaml │ │ ├── samsum │ │ └── templates.yaml │ │ ├── scan │ │ ├── addprim_jump │ │ │ └── templates.yaml │ │ ├── addprim_turn_left │ │ │ └── templates.yaml │ │ ├── filler_num0 │ │ │ └── templates.yaml │ │ ├── filler_num1 │ │ │ └── templates.yaml │ │ ├── filler_num2 │ │ │ └── templates.yaml │ │ ├── filler_num3 │ │ │ └── templates.yaml │ │ ├── length │ │ │ └── templates.yaml │ │ ├── simple │ │ │ └── templates.yaml │ │ ├── template_around_right │ │ │ └── templates.yaml │ │ ├── template_jump_around_right │ │ │ └── templates.yaml │ │ ├── template_opposite_right │ │ │ └── templates.yaml │ │ └── template_right │ │ │ └── templates.yaml │ │ ├── scicite │ │ └── templates.yaml │ │ ├── scientific_papers │ │ ├── arxiv │ │ │ └── templates.yaml │ │ └── pubmed │ │ │ └── templates.yaml │ │ ├── sciq │ │ └── templates.yaml │ │ ├── scitail │ │ ├── snli_format │ │ │ └── templates.yaml │ │ └── tsv_format │ │ │ └── templates.yaml │ │ ├── scitldr │ │ └── Abstract │ │ │ └── templates.yaml │ │ ├── selqa │ │ └── answer_selection_analysis │ │ │ └── templates.yaml │ │ ├── sem_eval_2010_task_8 │ │ └── templates.yaml │ │ ├── sem_eval_2014_task_1 │ │ └── templates.yaml │ │ ├── sent_comp │ │ └── templates.yaml │ │ ├── sick │ │ └── templates.yaml │ │ ├── sms_spam │ │ └── templates.yaml │ │ ├── snips_built_in_intents │ │ └── templates.yaml │ │ ├── snli │ │ └── templates.yaml │ │ ├── social_i_qa │ │ └── templates.yaml │ │ ├── species_800 │ │ └── templates.yaml │ │ ├── squad │ │ └── templates.yaml │ │ ├── squad_adversarial │ │ └── AddSent │ │ │ └── templates.yaml │ │ ├── squad_v2 │ │ └── templates.yaml │ │ ├── squadshifts │ │ ├── amazon │ │ │ └── templates.yaml │ │ ├── new_wiki │ │ │ └── templates.yaml │ │ └── nyt │ │ │ └── templates.yaml │ │ ├── sst │ │ └── default │ │ │ └── templates.yaml │ │ ├── sst2 │ │ └── templates.yaml │ │ ├── story_cloze │ │ └── 2016 │ │ │ └── templates.yaml │ │ ├── stsb_multi_mt │ │ └── en │ │ │ └── templates.yaml │ │ ├── subjqa │ │ ├── books │ │ │ └── templates.yaml │ │ ├── electronics │ │ │ └── templates.yaml │ │ ├── grocery │ │ │ └── templates.yaml │ │ ├── movies │ │ │ └── templates.yaml │ │ ├── restaurants │ │ │ └── templates.yaml │ │ └── tripadvisor │ │ │ └── templates.yaml │ │ ├── super_glue │ │ ├── axb │ │ │ └── templates.yaml │ │ ├── axg │ │ │ └── templates.yaml │ │ ├── boolq │ │ │ └── templates.yaml │ │ ├── cb │ │ │ └── templates.yaml │ │ ├── copa │ │ │ └── templates.yaml │ │ ├── multirc │ │ │ └── templates.yaml │ │ ├── record │ │ │ └── templates.yaml │ │ ├── rte │ │ │ └── templates.yaml │ │ ├── wic │ │ │ └── templates.yaml │ │ └── wsc.fixed │ │ │ └── templates.yaml │ │ ├── swag │ │ └── regular │ │ │ └── templates.yaml │ │ ├── tab_fact │ │ └── tab_fact │ │ │ └── templates.yaml │ │ ├── tmu_gfm_dataset │ │ └── templates.yaml │ │ ├── trec │ │ └── templates.yaml │ │ ├── trivia_qa │ │ └── unfiltered │ │ │ └── templates.yaml │ │ ├── turk │ │ └── templates.yaml │ │ ├── tweet_eval │ │ ├── emoji │ │ │ └── templates.yaml │ │ ├── emotion │ │ │ └── templates.yaml │ │ ├── hate │ │ │ └── templates.yaml │ │ ├── irony │ │ │ └── templates.yaml │ │ ├── offensive │ │ │ └── templates.yaml │ │ ├── sentiment │ │ │ └── templates.yaml │ │ ├── stance_abortion │ │ │ └── templates.yaml │ │ ├── stance_atheism │ │ │ └── templates.yaml │ │ ├── stance_climate │ │ │ └── templates.yaml │ │ ├── stance_feminist │ │ │ └── templates.yaml │ │ └── stance_hillary │ │ │ └── templates.yaml │ │ ├── tydiqa │ │ ├── primary_task │ │ │ └── templates.yaml │ │ └── secondary_task │ │ │ └── templates.yaml │ │ ├── web_questions │ │ └── templates.yaml │ │ ├── wiki_bio │ │ └── templates.yaml │ │ ├── wiki_hop │ │ ├── masked │ │ │ └── templates.yaml │ │ └── original │ │ │ └── templates.yaml │ │ ├── wiki_qa │ │ └── templates.yaml │ │ ├── wiki_split │ │ └── templates.yaml │ │ ├── wino_bias │ │ ├── type1_anti │ │ │ └── templates.yaml │ │ ├── type1_pro │ │ │ └── templates.yaml │ │ ├── type2_anti │ │ │ └── templates.yaml │ │ └── type2_pro │ │ │ └── templates.yaml │ │ ├── winograd_wsc │ │ ├── wsc273 │ │ │ └── templates.yaml │ │ └── wsc285 │ │ │ └── templates.yaml │ │ ├── winogrande │ │ ├── winogrande_debiased │ │ │ └── templates.yaml │ │ ├── winogrande_l │ │ │ └── templates.yaml │ │ ├── winogrande_m │ │ │ └── templates.yaml │ │ ├── winogrande_s │ │ │ └── templates.yaml │ │ ├── winogrande_xl │ │ │ └── templates.yaml │ │ └── winogrande_xs │ │ │ └── templates.yaml │ │ ├── wiqa │ │ └── templates.yaml │ │ ├── xnli │ │ └── en │ │ │ └── templates.yaml │ │ ├── xquad │ │ └── xquad.en │ │ │ └── templates.yaml │ │ ├── xquad_r │ │ └── en │ │ │ └── templates.yaml │ │ ├── xsum │ │ └── templates.yaml │ │ ├── yahoo_answers_qa │ │ └── templates.yaml │ │ ├── yahoo_answers_topics │ │ └── templates.yaml │ │ ├── yelp_polarity │ │ └── templates.yaml │ │ ├── yelp_review_full │ │ └── templates.yaml │ │ └── zest │ │ └── templates.yaml ├── resources │ ├── adjectives.json │ └── names.json ├── run.py ├── training │ ├── __init__.py │ ├── classifier.py │ ├── common.py │ ├── platt_scaling.py │ ├── supervised.py │ ├── sweep.py │ └── train.py └── utils │ ├── __init__.py │ ├── constants.py │ ├── data_utils.py │ ├── gpu_utils.py │ ├── hf_utils.py │ ├── math_util.py │ ├── pretty.py │ ├── tree_utils.py │ └── typing.py ├── pyproject.toml ├── setup.cfg └── tests ├── __init__.py ├── dbpedia_prompts.yaml ├── super_glue_prompts.yaml ├── test_classifier.py ├── test_encodings.py ├── test_gpu_example.py ├── test_inference_server.py ├── test_load_prompts.py ├── test_math.py ├── test_metrics.py ├── test_samplers.py ├── test_smoke_elicit.py └── test_smoke_eval.py /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/.devcontainer/Dockerfile -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/.devcontainer/devcontainer.json -------------------------------------------------------------------------------- /.github/workflows/cpu_ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/.github/workflows/cpu_ci.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/.vscode/settings.json -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/LICENSE.md -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/README.md -------------------------------------------------------------------------------- /elk/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/__init__.py -------------------------------------------------------------------------------- /elk/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/__main__.py -------------------------------------------------------------------------------- /elk/debug_logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/debug_logging.py -------------------------------------------------------------------------------- /elk/evaluation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/evaluation/__init__.py -------------------------------------------------------------------------------- /elk/evaluation/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/evaluation/evaluate.py -------------------------------------------------------------------------------- /elk/extraction/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/extraction/__init__.py -------------------------------------------------------------------------------- /elk/extraction/balanced_sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/extraction/balanced_sampler.py -------------------------------------------------------------------------------- /elk/extraction/dataset_name.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/extraction/dataset_name.py -------------------------------------------------------------------------------- /elk/extraction/extraction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/extraction/extraction.py -------------------------------------------------------------------------------- /elk/extraction/generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/extraction/generator.py -------------------------------------------------------------------------------- /elk/extraction/inference_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/extraction/inference_server.py -------------------------------------------------------------------------------- /elk/extraction/prompt_loading.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/extraction/prompt_loading.py -------------------------------------------------------------------------------- /elk/files.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/files.py -------------------------------------------------------------------------------- /elk/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/metrics/__init__.py -------------------------------------------------------------------------------- /elk/metrics/accuracy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/metrics/accuracy.py -------------------------------------------------------------------------------- /elk/metrics/calibration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/metrics/calibration.py -------------------------------------------------------------------------------- /elk/metrics/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/metrics/eval.py -------------------------------------------------------------------------------- /elk/metrics/roc_auc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/metrics/roc_auc.py -------------------------------------------------------------------------------- /elk/plotting/command.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/plotting/command.py -------------------------------------------------------------------------------- /elk/plotting/visualize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/plotting/visualize.py -------------------------------------------------------------------------------- /elk/promptsource/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/LICENSE -------------------------------------------------------------------------------- /elk/promptsource/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/__init__.py -------------------------------------------------------------------------------- /elk/promptsource/templates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates.py -------------------------------------------------------------------------------- /elk/promptsource/templates/Zaid/coqa_expanded/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/Zaid/coqa_expanded/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/Zaid/quac_expanded/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/Zaid/quac_expanded/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/_default/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/_default/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/_no_suffix/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/_no_suffix/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/acronym_identification/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/acronym_identification/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/adversarial_qa/dbert/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/adversarial_qa/dbert/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/adversarial_qa/dbidaf/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/adversarial_qa/dbidaf/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/adversarial_qa/droberta/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/adversarial_qa/droberta/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/aeslc/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/aeslc/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/ag_news/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/ag_news/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/amazon_polarity/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/amazon_polarity/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/amazon_reviews_multi/en/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/amazon_reviews_multi/en/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/ambig_qa/light/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/ambig_qa/light/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/anli/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/anli/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/app_reviews/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/app_reviews/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/aqua_rat/raw/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/aqua_rat/raw/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/art/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/art/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/asnq/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/asnq/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/asset/ratings/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/asset/ratings/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/asset/simplification/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/asset/simplification/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/banking77/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/banking77/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/billsum/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/billsum/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/bing_coronavirus_query_set/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/bing_coronavirus_query_set/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/biosses/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/biosses/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/blended_skill_talk/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/blended_skill_talk/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/boolq_pt/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/boolq_pt/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/cbt/CN/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/cbt/CN/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/cbt/NE/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/cbt/NE/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/cbt/P/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/cbt/P/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/cbt/V/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/cbt/V/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/cbt/raw/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/cbt/raw/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/cc_news/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/cc_news/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/christykoh/ag_news_pt/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/christykoh/ag_news_pt/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/christykoh/boolq_pt/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/christykoh/boolq_pt/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/christykoh/imdb_ar/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/christykoh/imdb_ar/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/christykoh/imdb_es/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/christykoh/imdb_es/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/christykoh/imdb_fr/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/christykoh/imdb_fr/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/christykoh/imdb_pt/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/christykoh/imdb_pt/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/christykoh/imdb_zh/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/christykoh/imdb_zh/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/circa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/circa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/climate_fever/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/climate_fever/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/codah/codah/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/codah/codah/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/codah/fold_0/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/codah/fold_0/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/codah/fold_1/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/codah/fold_1/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/codah/fold_2/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/codah/fold_2/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/codah/fold_3/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/codah/fold_3/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/codah/fold_4/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/codah/fold_4/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/common_gen/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/common_gen/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/commonsense_qa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/commonsense_qa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/conv_ai/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/conv_ai/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/conv_ai_2/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/conv_ai_2/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/conv_ai_3/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/conv_ai_3/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/coqa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/coqa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/cord19/metadata/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/cord19/metadata/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/cos_e/v1.0/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/cos_e/v1.0/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/cos_e/v1.11/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/cos_e/v1.11/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/cosmos_qa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/cosmos_qa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/covid_qa_castorini/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/covid_qa_castorini/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/craffel/openai_lambada/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/craffel/openai_lambada/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/craigslist_bargains/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/craigslist_bargains/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/crows_pairs/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/crows_pairs/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/dbpedia_14/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/dbpedia_14/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/discofuse/discofuse-sport/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/discofuse/discofuse-sport/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/discovery/discovery/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/discovery/discovery/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/docred/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/docred/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/dream/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/dream/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/drop/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/drop/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/duorc/ParaphraseRC/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/duorc/ParaphraseRC/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/duorc/SelfRC/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/duorc/SelfRC/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/e2e_nlg_cleaned/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/e2e_nlg_cleaned/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/emo/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/emo/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/emotion/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/emotion/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/enriched_web_nlg/en/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/enriched_web_nlg/en/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/esnli/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/esnli/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/fever/v1.0/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/fever/v1.0/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/fever/v2.0/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/fever/v2.0/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/freebase_qa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/freebase_qa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/generated_reviews_enth/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/generated_reviews_enth/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/gigaword/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/gigaword/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/ax/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/ax/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/cola/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/cola/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/mnli/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/mnli/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/mnli_matched/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/mnli_matched/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/mnli_mismatched/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/mnli_mismatched/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/mrpc/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/mrpc/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/qnli/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/qnli/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/qqp/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/qqp/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/rte/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/rte/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/sst2/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/sst2/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/stsb/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/stsb/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/glue/wnli/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/glue/wnli/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/google_wellformed_query/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/google_wellformed_query/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/great_code/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/great_code/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/gutenberg_time/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/gutenberg_time/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/hans/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/hans/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/hate_speech18/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/hate_speech18/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/head_qa/en/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/head_qa/en/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/health_fact/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/health_fact/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/hellaswag/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/hellaswag/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/hlgd/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/hlgd/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/hotpot_qa/distractor/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/hotpot_qa/distractor/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/hotpot_qa/fullwiki/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/hotpot_qa/fullwiki/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/humicroedit/subtask-1/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/humicroedit/subtask-1/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/humicroedit/subtask-2/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/humicroedit/subtask-2/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/imdb/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/imdb/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/jfleg/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/jfleg/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/jigsaw_unintended_bias/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/jigsaw_unintended_bias/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/kelm/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/kelm/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/kilt_tasks/nq/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/kilt_tasks/nq/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/lama/trex/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/lama/trex/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/lambada/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/lambada/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/lauritowal/redefine_math/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/lauritowal/redefine_math/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/liar/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/liar/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/limit/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/limit/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/math_qa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/math_qa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/mc_taco/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/mc_taco/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/mdd/task1_qa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/mdd/task1_qa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/mdd/task2_recs/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/mdd/task2_recs/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/mdd/task3_qarecs/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/mdd/task3_qarecs/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/medal/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/medal/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/medical_questions_pairs/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/medical_questions_pairs/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/meta_woz/dialogues/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/meta_woz/dialogues/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/mocha/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/mocha/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/movie_rationales/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/movie_rationales/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/multi_news/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/multi_news/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/multi_nli/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/multi_nli/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/multi_x_science_sum/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/multi_x_science_sum/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/mwsc/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/mwsc/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/narrativeqa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/narrativeqa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/ncbi_disease/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/ncbi_disease/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/newspop/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/newspop/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/nlu_evaluation_data/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/nlu_evaluation_data/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/nq_open/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/nq_open/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/numer_sense/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/numer_sense/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/onestop_english/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/onestop_english/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/openai_humaneval/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/openai_humaneval/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/openbookqa/additional/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/openbookqa/additional/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/openbookqa/main/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/openbookqa/main/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/paws-x/en/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/paws-x/en/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/paws/labeled_final/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/paws/labeled_final/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/paws/labeled_swap/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/paws/labeled_swap/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/paws/unlabeled_final/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/paws/unlabeled_final/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/piqa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/piqa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/poem_sentiment/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/poem_sentiment/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/qa_srl/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/qa_srl/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/qa_zre/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/qa_zre/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/qasc/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/qasc/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/qed/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/qed/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/quac/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/quac/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/quail/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/quail/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/quarel/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/quarel/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/quartz/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/quartz/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/quora/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/quora/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/quoref/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/quoref/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/race/all/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/race/all/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/race/high/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/race/high/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/race/middle/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/race/middle/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/reaganjlee/boolq_ar/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/reaganjlee/boolq_ar/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/reaganjlee/boolq_es/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/reaganjlee/boolq_es/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/reaganjlee/boolq_fr/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/reaganjlee/boolq_fr/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/reaganjlee/boolq_pt/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/reaganjlee/boolq_pt/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/reaganjlee/boolq_zh/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/reaganjlee/boolq_zh/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/riddle_sense/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/riddle_sense/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/ropes/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/ropes/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/rotten_tomatoes/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/rotten_tomatoes/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/samsum/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/samsum/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/addprim_jump/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/addprim_jump/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/addprim_turn_left/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/addprim_turn_left/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/filler_num0/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/filler_num0/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/filler_num1/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/filler_num1/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/filler_num2/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/filler_num2/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/filler_num3/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/filler_num3/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/length/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/length/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/simple/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/simple/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/template_around_right/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/template_around_right/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/template_jump_around_right/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/template_jump_around_right/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/template_opposite_right/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/template_opposite_right/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scan/template_right/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scan/template_right/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scicite/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scicite/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scientific_papers/arxiv/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scientific_papers/arxiv/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scientific_papers/pubmed/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scientific_papers/pubmed/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/sciq/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/sciq/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scitail/snli_format/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scitail/snli_format/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scitail/tsv_format/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scitail/tsv_format/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/scitldr/Abstract/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/scitldr/Abstract/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/selqa/answer_selection_analysis/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/selqa/answer_selection_analysis/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/sem_eval_2010_task_8/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/sem_eval_2010_task_8/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/sem_eval_2014_task_1/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/sem_eval_2014_task_1/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/sent_comp/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/sent_comp/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/sick/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/sick/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/sms_spam/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/sms_spam/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/snips_built_in_intents/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/snips_built_in_intents/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/snli/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/snli/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/social_i_qa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/social_i_qa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/species_800/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/species_800/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/squad/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/squad/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/squad_adversarial/AddSent/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/squad_adversarial/AddSent/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/squad_v2/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/squad_v2/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/squadshifts/amazon/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/squadshifts/amazon/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/squadshifts/new_wiki/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/squadshifts/new_wiki/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/squadshifts/nyt/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/squadshifts/nyt/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/sst/default/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/sst/default/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/sst2/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/sst2/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/story_cloze/2016/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/story_cloze/2016/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/stsb_multi_mt/en/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/stsb_multi_mt/en/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/subjqa/books/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/subjqa/books/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/subjqa/electronics/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/subjqa/electronics/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/subjqa/grocery/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/subjqa/grocery/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/subjqa/movies/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/subjqa/movies/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/subjqa/restaurants/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/subjqa/restaurants/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/subjqa/tripadvisor/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/subjqa/tripadvisor/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/super_glue/axb/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/super_glue/axb/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/super_glue/axg/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/super_glue/axg/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/super_glue/boolq/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/super_glue/boolq/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/super_glue/cb/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/super_glue/cb/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/super_glue/copa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/super_glue/copa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/super_glue/multirc/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/super_glue/multirc/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/super_glue/record/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/super_glue/record/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/super_glue/rte/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/super_glue/rte/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/super_glue/wic/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/super_glue/wic/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/super_glue/wsc.fixed/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/super_glue/wsc.fixed/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/swag/regular/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/swag/regular/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tab_fact/tab_fact/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tab_fact/tab_fact/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tmu_gfm_dataset/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tmu_gfm_dataset/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/trec/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/trec/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/trivia_qa/unfiltered/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/trivia_qa/unfiltered/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/turk/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/turk/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tweet_eval/emoji/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tweet_eval/emoji/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tweet_eval/emotion/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tweet_eval/emotion/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tweet_eval/hate/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tweet_eval/hate/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tweet_eval/irony/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tweet_eval/irony/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tweet_eval/offensive/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tweet_eval/offensive/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tweet_eval/sentiment/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tweet_eval/sentiment/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tweet_eval/stance_abortion/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tweet_eval/stance_abortion/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tweet_eval/stance_atheism/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tweet_eval/stance_atheism/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tweet_eval/stance_climate/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tweet_eval/stance_climate/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tweet_eval/stance_feminist/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tweet_eval/stance_feminist/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tweet_eval/stance_hillary/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tweet_eval/stance_hillary/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tydiqa/primary_task/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tydiqa/primary_task/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/tydiqa/secondary_task/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/tydiqa/secondary_task/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/web_questions/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/web_questions/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/wiki_bio/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/wiki_bio/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/wiki_hop/masked/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/wiki_hop/masked/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/wiki_hop/original/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/wiki_hop/original/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/wiki_qa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/wiki_qa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/wiki_split/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/wiki_split/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/wino_bias/type1_anti/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/wino_bias/type1_anti/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/wino_bias/type1_pro/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/wino_bias/type1_pro/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/wino_bias/type2_anti/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/wino_bias/type2_anti/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/wino_bias/type2_pro/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/wino_bias/type2_pro/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/winograd_wsc/wsc273/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/winograd_wsc/wsc273/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/winograd_wsc/wsc285/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/winograd_wsc/wsc285/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/winogrande/winogrande_debiased/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/winogrande/winogrande_debiased/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/winogrande/winogrande_l/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/winogrande/winogrande_l/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/winogrande/winogrande_m/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/winogrande/winogrande_m/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/winogrande/winogrande_s/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/winogrande/winogrande_s/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/winogrande/winogrande_xl/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/winogrande/winogrande_xl/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/winogrande/winogrande_xs/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/winogrande/winogrande_xs/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/wiqa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/wiqa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/xnli/en/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/xnli/en/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/xquad/xquad.en/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/xquad/xquad.en/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/xquad_r/en/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/xquad_r/en/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/xsum/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/xsum/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/yahoo_answers_qa/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/yahoo_answers_qa/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/yahoo_answers_topics/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/yahoo_answers_topics/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/yelp_polarity/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/yelp_polarity/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/yelp_review_full/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/yelp_review_full/templates.yaml -------------------------------------------------------------------------------- /elk/promptsource/templates/zest/templates.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/promptsource/templates/zest/templates.yaml -------------------------------------------------------------------------------- /elk/resources/adjectives.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/resources/adjectives.json -------------------------------------------------------------------------------- /elk/resources/names.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/resources/names.json -------------------------------------------------------------------------------- /elk/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/run.py -------------------------------------------------------------------------------- /elk/training/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/training/__init__.py -------------------------------------------------------------------------------- /elk/training/classifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/training/classifier.py -------------------------------------------------------------------------------- /elk/training/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/training/common.py -------------------------------------------------------------------------------- /elk/training/platt_scaling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/training/platt_scaling.py -------------------------------------------------------------------------------- /elk/training/supervised.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/training/supervised.py -------------------------------------------------------------------------------- /elk/training/sweep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/training/sweep.py -------------------------------------------------------------------------------- /elk/training/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/training/train.py -------------------------------------------------------------------------------- /elk/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/utils/__init__.py -------------------------------------------------------------------------------- /elk/utils/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/utils/constants.py -------------------------------------------------------------------------------- /elk/utils/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/utils/data_utils.py -------------------------------------------------------------------------------- /elk/utils/gpu_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/utils/gpu_utils.py -------------------------------------------------------------------------------- /elk/utils/hf_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/utils/hf_utils.py -------------------------------------------------------------------------------- /elk/utils/math_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/utils/math_util.py -------------------------------------------------------------------------------- /elk/utils/pretty.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/utils/pretty.py -------------------------------------------------------------------------------- /elk/utils/tree_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/utils/tree_utils.py -------------------------------------------------------------------------------- /elk/utils/typing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/elk/utils/typing.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/pyproject.toml -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/dbpedia_prompts.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/dbpedia_prompts.yaml -------------------------------------------------------------------------------- /tests/super_glue_prompts.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/super_glue_prompts.yaml -------------------------------------------------------------------------------- /tests/test_classifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/test_classifier.py -------------------------------------------------------------------------------- /tests/test_encodings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/test_encodings.py -------------------------------------------------------------------------------- /tests/test_gpu_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/test_gpu_example.py -------------------------------------------------------------------------------- /tests/test_inference_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/test_inference_server.py -------------------------------------------------------------------------------- /tests/test_load_prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/test_load_prompts.py -------------------------------------------------------------------------------- /tests/test_math.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/test_math.py -------------------------------------------------------------------------------- /tests/test_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/test_metrics.py -------------------------------------------------------------------------------- /tests/test_samplers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/test_samplers.py -------------------------------------------------------------------------------- /tests/test_smoke_elicit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/test_smoke_elicit.py -------------------------------------------------------------------------------- /tests/test_smoke_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/elk/HEAD/tests/test_smoke_eval.py --------------------------------------------------------------------------------