├── .coveragerc ├── .flake8 ├── .github └── workflows │ ├── pull_request.yml │ └── python-app.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CITATION.bib ├── CODEOWNERS ├── LICENSE.md ├── README.md ├── docs ├── decontamination.md ├── description_guide.md ├── img │ └── fewshot_example_gpt3.png ├── jptasks.md ├── prompt_templates.md ├── task_guide.md └── task_table.md ├── ignore.txt ├── lm_eval ├── __init__.py ├── base.py ├── datasets │ ├── README.md │ ├── __init__.py │ ├── asdiv │ │ ├── __init__.py │ │ ├── asdiv.py │ │ └── dataset_infos.json │ ├── coqa │ │ ├── __init__.py │ │ ├── coqa.py │ │ └── dataset_infos.json │ ├── drop │ │ ├── __init__.py │ │ ├── dataset_infos.json │ │ └── drop.py │ ├── headqa │ │ ├── __init__.py │ │ ├── dataset_infos.json │ │ └── headqa.py │ ├── hendrycks_ethics │ │ ├── __init__.py │ │ ├── dataset_infos.json │ │ └── hendrycks_ethics.py │ ├── hendrycks_math │ │ ├── __init__.py │ │ ├── dataset_infos.json │ │ └── hendrycks_math.py │ ├── lambada_ja │ │ ├── __init__.py │ │ └── lambada_ja.py │ ├── logiqa │ │ ├── __init__.py │ │ ├── dataset_infos.json │ │ └── logiqa.py │ ├── mutual │ │ ├── __init__.py │ │ ├── dataset_infos.json │ │ └── mutual.py │ ├── pile │ │ ├── __init__.py │ │ ├── dataset_infos.json │ │ └── pile.py │ ├── quac │ │ ├── __init__.py │ │ ├── dataset_infos.json │ │ └── quac.py │ ├── sat_analogies │ │ ├── __init__.py │ │ └── sat_analogies.py │ ├── triviaqa │ │ ├── README.md │ │ ├── __init__.py │ │ ├── dataset_infos.json │ │ └── triviaqa.py │ └── unscramble │ │ ├── __init__.py │ │ ├── dataset_infos.json │ │ └── unscramble.py ├── decontamination │ ├── __init__.py │ ├── archiver.py │ ├── decontaminate.py │ └── janitor.py ├── evaluator.py ├── jasquad │ ├── README.md │ ├── __init__.py │ ├── evaluate.py │ ├── jasquad.py │ └── requirements.txt ├── metrics.py ├── models │ ├── __init__.py │ ├── dummy.py │ ├── gpt2.py │ ├── gpt3.py │ ├── huggingface.py │ └── textsynth.py ├── prompts.py ├── suites │ ├── __init__.py │ └── configs │ │ └── ja8.conf ├── tasks │ ├── __init__.py │ ├── anli.py │ ├── arc.py │ ├── arithmetic.py │ ├── asdiv.py │ ├── blimp.py │ ├── cbt.py │ ├── coqa.py │ ├── crowspairs.py │ ├── drop.py │ ├── glue.py │ ├── gsm8k.py │ ├── headqa.py │ ├── hellaswag.py │ ├── hendrycks_ethics.py │ ├── hendrycks_math.py │ ├── hendrycks_test.py │ ├── ja │ │ ├── __init__.py │ │ ├── jaqket_v1.py │ │ ├── jaqket_v2.py │ │ ├── jaquad.py │ │ ├── jblimp.py │ │ ├── jcola.py │ │ ├── jcommonsenseqa.py │ │ ├── jnli.py │ │ ├── jsquad.py │ │ ├── marc_ja.py │ │ ├── mgsm.py │ │ ├── wikilingua_ja.py │ │ ├── xlsum_ja.py │ │ └── xwinograd_ja.py │ ├── lambada.py │ ├── lambada_cloze.py │ ├── lambada_multilingual.py │ ├── logiqa.py │ ├── mathqa.py │ ├── mc_taco.py │ ├── mutual.py │ ├── naturalqs.py │ ├── openbookqa.py │ ├── pile.py │ ├── piqa.py │ ├── prost.py │ ├── pubmedqa.py │ ├── qa4mre.py │ ├── qasper.py │ ├── quac.py │ ├── race.py │ ├── sat.py │ ├── sciq.py │ ├── squad.py │ ├── storycloze.py │ ├── superglue.py │ ├── swag.py │ ├── toxigen.py │ ├── translation.py │ ├── triviaqa.py │ ├── truthfulqa.py │ ├── unscramble.py │ ├── webqs.py │ ├── wikitext.py │ ├── winogrande.py │ └── wsc273.py └── utils.py ├── main.py ├── models ├── abeja-gpt-neox-japanese-2.7b │ ├── harness.jsquad-1.2.sh │ ├── harness.sh │ ├── result.json │ └── result.jsquad-1.2.json ├── community │ ├── cyberagent-open-calm-instruct-1b_1.3.0 │ │ ├── README.md │ │ ├── harness.sh │ │ └── result.json │ ├── cyberagent-open-calm-instruct-3b_1.3.0 │ │ ├── README.md │ │ ├── harness.sh │ │ └── result.json │ ├── cyberagent-open-calm-instruct-7b_1.5.4 │ │ └── harness.sh │ ├── cyberagent-open-calm-instruct-7b_1.9.4 │ │ ├── harness.sh │ │ ├── result.json │ │ ├── result_2.json │ │ └── xwinograd_ja.result.json │ └── rinna-instruct-1b_0.1.0 │ │ ├── harness.sh │ │ └── result.json ├── cyberagent │ ├── cyberagent-open-calm-1b │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ ├── result.jsquad-1.2.json │ │ └── result.mgsm.json │ ├── cyberagent-open-calm-3b │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ ├── result.jsquad-1.2.json │ │ └── result.mgsm.json │ ├── cyberagent-open-calm-7b │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ ├── result.jsquad-1.2.json │ │ └── result.mgsm.json │ ├── cyberagent-open-calm-large │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ └── result.jsquad-1.2.json │ └── cyberagent-open-calm-medium │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ └── result.jsquad-1.2.json ├── harness.conf ├── line-corporation │ ├── line-corporation-japanese-large-lm-1.7b-instruction-sft │ │ ├── harness.sh │ │ └── result.json │ ├── line-corporation-japanese-large-lm-1.7b │ │ ├── harness.sh │ │ └── result.json │ ├── line-corporation-japanese-large-lm-3.6b-instruction-sft │ │ ├── harness.sh │ │ └── result.json │ └── line-corporation-japanese-large-lm-3.6b │ │ ├── harness.sh │ │ └── result.json ├── llama │ ├── llama-13b │ │ ├── harness.sh │ │ └── result.json │ ├── llama-30b │ │ ├── harness.sh │ │ └── result.json │ ├── llama-65b │ │ ├── harness.sh │ │ └── result.json │ └── llama-7b │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ └── result.jsquad-1.2.json ├── llama2 │ ├── llama2-13b-chat │ │ ├── harness.sh │ │ └── result.json │ ├── llama2-13b │ │ ├── harness.sh │ │ └── result.json │ ├── llama2-7b-chat │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ └── result.jsquad-1.2.json │ └── llama2-7b │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ └── result.jsquad-1.2.json ├── openai │ └── gpt3 │ │ └── result.mgsm.json ├── rinna │ ├── harness.conf │ ├── rinna-bilingual-gpt-neox-4b-instruction-ppo │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ └── result.jsquad-1.2.json │ ├── rinna-bilingual-gpt-neox-4b-instruction-sft │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ └── result.jsquad-1.2.json │ ├── rinna-bilingual-gpt-neox-4b │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ └── result.jsquad-1.2.json │ ├── rinna-japanese-gpt-1b │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ └── result.jsquad-1.2.json │ ├── rinna-japanese-gpt-neox-3.6b-instruction-ppo │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ ├── result.jsquad-1.2.json │ │ └── result.mgsm.json │ ├── rinna-japanese-gpt-neox-3.6b-instruction-sft-v2 │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ ├── result.jsquad-1.2.json │ │ └── result.mgsm.json │ ├── rinna-japanese-gpt-neox-3.6b-instruction-sft │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ └── result.jsquad-1.2.json │ ├── rinna-japanese-gpt-neox-3.6b │ │ ├── harness.jsquad-1.2.sh │ │ ├── harness.sh │ │ ├── result.json │ │ ├── result.jsquad-1.2.json │ │ └── result.mgsm.json │ └── rinna-japanese-gpt-neox-small │ │ ├── harness.sh │ │ └── result.json ├── stabilityai │ ├── experiments │ │ ├── stablelm-jp-1b-compact-v1 │ │ │ ├── harness.sh │ │ │ └── result.json │ │ ├── stablelm-jp-1b-jav1-sl2k-300b │ │ │ ├── harness.sh │ │ │ └── result.json │ │ ├── stablelm-jp-1b-jav1-sl2k-slw-300b │ │ │ ├── harness.sh │ │ │ └── result.json │ │ ├── stablelm-jp-1b-jav1_rp-sl2k-slw-300b │ │ │ ├── harness.sh │ │ │ └── result.json │ │ ├── stablelm-jp-1b-rp_then_jav1-294b │ │ │ ├── harness.sh │ │ │ └── result.json │ │ ├── stablelm-jp-3b-ja50_rp50-700b │ │ │ ├── harness_template-0.1.sh │ │ │ ├── harness_template-0.2.sh │ │ │ ├── result_template-0.1.json │ │ │ └── result_template-0.2.json │ │ ├── stablelm-jp-instruct-1b_1.1.0 │ │ │ ├── harness.sh │ │ │ └── result.json │ │ ├── stablelm-jp-instruct-1b_1.3.0 │ │ │ ├── harness.sh │ │ │ └── result.json │ │ ├── stablelm-jp-instruct-1b_1.3.2 │ │ │ ├── harness.sh │ │ │ └── result.json │ │ ├── stablelm-jp-instruct-1b_1.5.2 │ │ │ ├── harness.sh │ │ │ └── result.json │ │ ├── stablelm-jp-instruct-1b_1.6.2 │ │ │ ├── harness.sh │ │ │ └── result.json │ │ ├── stablelm-jp-instruct-3b_1.3.0 │ │ │ ├── harness.sh │ │ │ ├── result.2.json │ │ │ ├── result.json │ │ │ └── xwinograd_ja.result.json │ │ ├── stablelm-jp-instruct-3b_1.5.0 │ │ │ ├── harness.sh │ │ │ └── result.json │ │ └── stablelm-jp-instruct-3b_1.5.2 │ │ │ ├── harness.sh │ │ │ ├── jaqket_mgsm.result.json │ │ │ ├── result.json │ │ │ └── xwinograd_ja.result.json │ ├── stabilityai-japanese-stablelm-base-alpha-7b │ │ ├── harness.sh │ │ └── result.json │ └── stabilityai-japanese-stablelm-instruct-alpha-7b │ │ ├── harness.sh │ │ └── result.json └── stablelm │ ├── harness.conf │ └── stablelm-jp-3b-ja50_rp50-700b │ └── harness.conf ├── pile_statistics.json ├── requirements-ja.txt ├── requirements.txt ├── scripts ├── __init__.py ├── clean_training_data │ ├── README.md │ ├── __init__.py │ ├── compress_and_package.py │ ├── generate_13_grams.py │ ├── investigate_pile.py │ ├── janitor_util.cpp │ ├── process_sorted_buckets.py │ └── sort_13_gram_buckets.py ├── compute_average_from_json.py ├── cost_estimate.py ├── generate_harness.py ├── get_prompts.py ├── harness_example.py ├── main_eval.py ├── make_gpt2_test_cases.py ├── make_leaderboard.py ├── make_table_tasks.py ├── merge_json.py ├── models.txt ├── notify.py ├── run_eval.py ├── run_suite.py ├── run_task.sh ├── run_task_batch.sh ├── run_task_for_models.sh └── write_out.py ├── setup.py ├── templates ├── new_multiple_choice_task.py └── new_task.py └── tests ├── test_description_dict.py ├── test_evaluator.py ├── test_generate_13_grams.py ├── test_janitor.py ├── test_misc.py ├── test_models.py ├── test_tasks.py ├── test_utils.py ├── test_version_stable.py └── testdata ├── anagrams1-v0-greedy_until ├── anagrams1-v0-res.json ├── anagrams2-v0-greedy_until ├── anagrams2-v0-res.json ├── anli_r1-v0-loglikelihood ├── anli_r1-v0-res.json ├── anli_r2-v0-loglikelihood ├── anli_r2-v0-res.json ├── anli_r3-v0-loglikelihood ├── anli_r3-v0-res.json ├── arc_challenge-v0-loglikelihood ├── arc_challenge-v0-res.json ├── arc_easy-v0-loglikelihood ├── arc_easy-v0-res.json ├── arithmetic_1dc-v0-loglikelihood ├── arithmetic_1dc-v0-res.json ├── arithmetic_2da-v0-loglikelihood ├── arithmetic_2da-v0-res.json ├── arithmetic_2dm-v0-loglikelihood ├── arithmetic_2dm-v0-res.json ├── arithmetic_2ds-v0-loglikelihood ├── arithmetic_2ds-v0-res.json ├── arithmetic_3da-v0-loglikelihood ├── arithmetic_3da-v0-res.json ├── arithmetic_3ds-v0-loglikelihood ├── arithmetic_3ds-v0-res.json ├── arithmetic_4da-v0-loglikelihood ├── arithmetic_4da-v0-res.json ├── arithmetic_4ds-v0-loglikelihood ├── arithmetic_4ds-v0-res.json ├── arithmetic_5da-v0-loglikelihood ├── arithmetic_5da-v0-res.json ├── arithmetic_5ds-v0-loglikelihood ├── arithmetic_5ds-v0-res.json ├── blimp_adjunct_island-v0-loglikelihood ├── blimp_adjunct_island-v0-res.json ├── blimp_anaphor_gender_agreement-v0-loglikelihood ├── blimp_anaphor_gender_agreement-v0-res.json ├── blimp_anaphor_number_agreement-v0-loglikelihood ├── blimp_anaphor_number_agreement-v0-res.json ├── blimp_animate_subject_passive-v0-loglikelihood ├── blimp_animate_subject_passive-v0-res.json ├── blimp_animate_subject_trans-v0-loglikelihood ├── blimp_animate_subject_trans-v0-res.json ├── blimp_causative-v0-loglikelihood ├── blimp_causative-v0-res.json ├── blimp_complex_NP_island-v0-loglikelihood ├── blimp_complex_NP_island-v0-res.json ├── blimp_coordinate_structure_constraint_complex_left_branch-v0-loglikelihood ├── blimp_coordinate_structure_constraint_complex_left_branch-v0-res.json ├── blimp_coordinate_structure_constraint_object_extraction-v0-loglikelihood ├── blimp_coordinate_structure_constraint_object_extraction-v0-res.json ├── blimp_determiner_noun_agreement_1-v0-loglikelihood ├── blimp_determiner_noun_agreement_1-v0-res.json ├── blimp_determiner_noun_agreement_2-v0-loglikelihood ├── blimp_determiner_noun_agreement_2-v0-res.json ├── blimp_determiner_noun_agreement_irregular_1-v0-loglikelihood ├── blimp_determiner_noun_agreement_irregular_1-v0-res.json ├── blimp_determiner_noun_agreement_irregular_2-v0-loglikelihood ├── blimp_determiner_noun_agreement_irregular_2-v0-res.json ├── blimp_determiner_noun_agreement_with_adj_2-v0-loglikelihood ├── blimp_determiner_noun_agreement_with_adj_2-v0-res.json ├── blimp_determiner_noun_agreement_with_adj_irregular_1-v0-loglikelihood ├── blimp_determiner_noun_agreement_with_adj_irregular_1-v0-res.json ├── blimp_determiner_noun_agreement_with_adj_irregular_2-v0-loglikelihood ├── blimp_determiner_noun_agreement_with_adj_irregular_2-v0-res.json ├── blimp_determiner_noun_agreement_with_adjective_1-v0-loglikelihood ├── blimp_determiner_noun_agreement_with_adjective_1-v0-res.json ├── blimp_distractor_agreement_relational_noun-v0-loglikelihood ├── blimp_distractor_agreement_relational_noun-v0-res.json ├── blimp_distractor_agreement_relative_clause-v0-loglikelihood ├── blimp_distractor_agreement_relative_clause-v0-res.json ├── blimp_drop_argument-v0-loglikelihood ├── blimp_drop_argument-v0-res.json ├── blimp_ellipsis_n_bar_1-v0-loglikelihood ├── blimp_ellipsis_n_bar_1-v0-res.json ├── blimp_ellipsis_n_bar_2-v0-loglikelihood ├── blimp_ellipsis_n_bar_2-v0-res.json ├── blimp_existential_there_object_raising-v0-loglikelihood ├── blimp_existential_there_object_raising-v0-res.json ├── blimp_existential_there_quantifiers_1-v0-loglikelihood ├── blimp_existential_there_quantifiers_1-v0-res.json ├── blimp_existential_there_quantifiers_2-v0-loglikelihood ├── blimp_existential_there_quantifiers_2-v0-res.json ├── blimp_existential_there_subject_raising-v0-loglikelihood ├── blimp_existential_there_subject_raising-v0-res.json ├── blimp_expletive_it_object_raising-v0-loglikelihood ├── blimp_expletive_it_object_raising-v0-res.json ├── blimp_inchoative-v0-loglikelihood ├── blimp_inchoative-v0-res.json ├── blimp_intransitive-v0-loglikelihood ├── blimp_intransitive-v0-res.json ├── blimp_irregular_past_participle_adjectives-v0-loglikelihood ├── blimp_irregular_past_participle_adjectives-v0-res.json ├── blimp_irregular_past_participle_verbs-v0-loglikelihood ├── blimp_irregular_past_participle_verbs-v0-res.json ├── blimp_irregular_plural_subject_verb_agreement_1-v0-loglikelihood ├── blimp_irregular_plural_subject_verb_agreement_1-v0-res.json ├── blimp_irregular_plural_subject_verb_agreement_2-v0-loglikelihood ├── blimp_irregular_plural_subject_verb_agreement_2-v0-res.json ├── blimp_left_branch_island_echo_question-v0-loglikelihood ├── blimp_left_branch_island_echo_question-v0-res.json ├── blimp_left_branch_island_simple_question-v0-loglikelihood ├── blimp_left_branch_island_simple_question-v0-res.json ├── blimp_matrix_question_npi_licensor_present-v0-loglikelihood ├── blimp_matrix_question_npi_licensor_present-v0-res.json ├── blimp_npi_present_1-v0-loglikelihood ├── blimp_npi_present_1-v0-res.json ├── blimp_npi_present_2-v0-loglikelihood ├── blimp_npi_present_2-v0-res.json ├── blimp_only_npi_licensor_present-v0-loglikelihood ├── blimp_only_npi_licensor_present-v0-res.json ├── blimp_only_npi_scope-v0-loglikelihood ├── blimp_only_npi_scope-v0-res.json ├── blimp_passive_1-v0-loglikelihood ├── blimp_passive_1-v0-res.json ├── blimp_passive_2-v0-loglikelihood ├── blimp_passive_2-v0-res.json ├── blimp_principle_A_c_command-v0-loglikelihood ├── blimp_principle_A_c_command-v0-res.json ├── blimp_principle_A_case_1-v0-loglikelihood ├── blimp_principle_A_case_1-v0-res.json ├── blimp_principle_A_case_2-v0-loglikelihood ├── blimp_principle_A_case_2-v0-res.json ├── blimp_principle_A_domain_1-v0-loglikelihood ├── blimp_principle_A_domain_1-v0-res.json ├── blimp_principle_A_domain_2-v0-loglikelihood ├── blimp_principle_A_domain_2-v0-res.json ├── blimp_principle_A_domain_3-v0-loglikelihood ├── blimp_principle_A_domain_3-v0-res.json ├── blimp_principle_A_reconstruction-v0-loglikelihood ├── blimp_principle_A_reconstruction-v0-res.json ├── blimp_regular_plural_subject_verb_agreement_1-v0-loglikelihood ├── blimp_regular_plural_subject_verb_agreement_1-v0-res.json ├── blimp_regular_plural_subject_verb_agreement_2-v0-loglikelihood ├── blimp_regular_plural_subject_verb_agreement_2-v0-res.json ├── blimp_sentential_negation_npi_licensor_present-v0-loglikelihood ├── blimp_sentential_negation_npi_licensor_present-v0-res.json ├── blimp_sentential_negation_npi_scope-v0-loglikelihood ├── blimp_sentential_negation_npi_scope-v0-res.json ├── blimp_sentential_subject_island-v0-loglikelihood ├── blimp_sentential_subject_island-v0-res.json ├── blimp_superlative_quantifiers_1-v0-loglikelihood ├── blimp_superlative_quantifiers_1-v0-res.json ├── blimp_superlative_quantifiers_2-v0-loglikelihood ├── blimp_superlative_quantifiers_2-v0-res.json ├── blimp_tough_vs_raising_1-v0-loglikelihood ├── blimp_tough_vs_raising_1-v0-res.json ├── blimp_tough_vs_raising_2-v0-loglikelihood ├── blimp_tough_vs_raising_2-v0-res.json ├── blimp_transitive-v0-loglikelihood ├── blimp_transitive-v0-res.json ├── blimp_wh_island-v0-loglikelihood ├── blimp_wh_island-v0-res.json ├── blimp_wh_questions_object_gap-v0-loglikelihood ├── blimp_wh_questions_object_gap-v0-res.json ├── blimp_wh_questions_subject_gap-v0-loglikelihood ├── blimp_wh_questions_subject_gap-v0-res.json ├── blimp_wh_questions_subject_gap_long_distance-v0-loglikelihood ├── blimp_wh_questions_subject_gap_long_distance-v0-res.json ├── blimp_wh_vs_that_no_gap-v0-loglikelihood ├── blimp_wh_vs_that_no_gap-v0-res.json ├── blimp_wh_vs_that_no_gap_long_distance-v0-loglikelihood ├── blimp_wh_vs_that_no_gap_long_distance-v0-res.json ├── blimp_wh_vs_that_with_gap-v0-loglikelihood ├── blimp_wh_vs_that_with_gap-v0-res.json ├── blimp_wh_vs_that_with_gap_long_distance-v0-loglikelihood ├── blimp_wh_vs_that_with_gap_long_distance-v0-res.json ├── boolq-v0-loglikelihood ├── boolq-v0-res.json ├── boolq-v1-loglikelihood ├── boolq-v1-res.json ├── cb-v0-loglikelihood ├── cb-v0-res.json ├── cb-v1-loglikelihood ├── cb-v1-res.json ├── cola-v0-loglikelihood ├── cola-v0-res.json ├── copa-v0-loglikelihood ├── copa-v0-res.json ├── coqa-v0-greedy_until ├── coqa-v0-res.json ├── coqa-v1-greedy_until ├── coqa-v1-res.json ├── crows_pairs_english-v0-loglikelihood ├── crows_pairs_english-v0-res.json ├── crows_pairs_english_age-v0-loglikelihood ├── crows_pairs_english_age-v0-res.json ├── crows_pairs_english_autre-v0-loglikelihood ├── crows_pairs_english_autre-v0-res.json ├── crows_pairs_english_disability-v0-loglikelihood ├── crows_pairs_english_disability-v0-res.json ├── crows_pairs_english_gender-v0-loglikelihood ├── crows_pairs_english_gender-v0-res.json ├── crows_pairs_english_nationality-v0-loglikelihood ├── crows_pairs_english_nationality-v0-res.json ├── crows_pairs_english_physical_appearance-v0-loglikelihood ├── crows_pairs_english_physical_appearance-v0-res.json ├── crows_pairs_english_race_color-v0-loglikelihood ├── crows_pairs_english_race_color-v0-res.json ├── crows_pairs_english_religion-v0-loglikelihood ├── crows_pairs_english_religion-v0-res.json ├── crows_pairs_english_sexual_orientation-v0-loglikelihood ├── crows_pairs_english_sexual_orientation-v0-res.json ├── crows_pairs_english_socioeconomic-v0-loglikelihood ├── crows_pairs_english_socioeconomic-v0-res.json ├── crows_pairs_french-v0-loglikelihood ├── crows_pairs_french-v0-res.json ├── crows_pairs_french_age-v0-loglikelihood ├── crows_pairs_french_age-v0-res.json ├── crows_pairs_french_autre-v0-loglikelihood ├── crows_pairs_french_autre-v0-res.json ├── crows_pairs_french_disability-v0-loglikelihood ├── crows_pairs_french_disability-v0-res.json ├── crows_pairs_french_gender-v0-loglikelihood ├── crows_pairs_french_gender-v0-res.json ├── crows_pairs_french_nationality-v0-loglikelihood ├── crows_pairs_french_nationality-v0-res.json ├── crows_pairs_french_physical_appearance-v0-loglikelihood ├── crows_pairs_french_physical_appearance-v0-res.json ├── crows_pairs_french_race_color-v0-loglikelihood ├── crows_pairs_french_race_color-v0-res.json ├── crows_pairs_french_religion-v0-loglikelihood ├── crows_pairs_french_religion-v0-res.json ├── crows_pairs_french_sexual_orientation-v0-loglikelihood ├── crows_pairs_french_sexual_orientation-v0-res.json ├── crows_pairs_french_socioeconomic-v0-loglikelihood ├── crows_pairs_french_socioeconomic-v0-res.json ├── cycle_letters-v0-greedy_until ├── cycle_letters-v0-res.json ├── drop-v0-greedy_until ├── drop-v0-res.json ├── drop-v1-greedy_until ├── drop-v1-res.json ├── ethics_cm-v0-loglikelihood ├── ethics_cm-v0-res.json ├── ethics_deontology-v0-loglikelihood ├── ethics_deontology-v0-res.json ├── ethics_justice-v0-loglikelihood ├── ethics_justice-v0-res.json ├── ethics_utilitarianism-v0-loglikelihood ├── ethics_utilitarianism-v0-res.json ├── ethics_utilitarianism_original-v0-loglikelihood ├── ethics_utilitarianism_original-v0-res.json ├── ethics_virtue-v0-loglikelihood ├── ethics_virtue-v0-res.json ├── gpt3_test_0deb8e9bde8e8327bbc48157f638ff3ba06b0cd816dad2beb8ad90f7fbe795c7.pkl ├── gpt3_test_8025023377febbd8c5f2b9f26705c394ff375d0cad7c89c10fd9b8e1eb66ff1c.pkl ├── gpt3_test_bb2cc49115e88788ed870ad0716eb00b280a885f91c7ed6e1e864435e5e2b6ac.pkl ├── gpt3_test_cfd11f555a5a63b6dfa114a55a932e51b724cdd44d4842586b9ce37260bf7aaa.pkl ├── gpt3_test_f307d52964c295e2005c5e782b688c24388e0cecadf29f1e6fc7f394236ea9c0.pkl ├── gsm8k-v0-greedy_until ├── gsm8k-v0-res.json ├── headqa-v0-loglikelihood ├── headqa-v0-res.json ├── headqa_en-v0-loglikelihood ├── headqa_en-v0-res.json ├── headqa_es-v0-loglikelihood ├── headqa_es-v0-res.json ├── hellaswag-v0-loglikelihood ├── hellaswag-v0-res.json ├── hendrycksTest-abstract_algebra-v0-loglikelihood ├── hendrycksTest-abstract_algebra-v0-res.json ├── hendrycksTest-anatomy-v0-loglikelihood ├── hendrycksTest-anatomy-v0-res.json ├── hendrycksTest-astronomy-v0-loglikelihood ├── hendrycksTest-astronomy-v0-res.json ├── hendrycksTest-business_ethics-v0-loglikelihood ├── hendrycksTest-business_ethics-v0-res.json ├── hendrycksTest-clinical_knowledge-v0-loglikelihood ├── hendrycksTest-clinical_knowledge-v0-res.json ├── hendrycksTest-college_biology-v0-loglikelihood ├── hendrycksTest-college_biology-v0-res.json ├── hendrycksTest-college_chemistry-v0-loglikelihood ├── hendrycksTest-college_chemistry-v0-res.json ├── hendrycksTest-college_computer_science-v0-loglikelihood ├── hendrycksTest-college_computer_science-v0-res.json ├── hendrycksTest-college_mathematics-v0-loglikelihood ├── hendrycksTest-college_mathematics-v0-res.json ├── hendrycksTest-college_medicine-v0-loglikelihood ├── hendrycksTest-college_medicine-v0-res.json ├── hendrycksTest-college_physics-v0-loglikelihood ├── hendrycksTest-college_physics-v0-res.json ├── hendrycksTest-computer_security-v0-loglikelihood ├── hendrycksTest-computer_security-v0-res.json ├── hendrycksTest-conceptual_physics-v0-loglikelihood ├── hendrycksTest-conceptual_physics-v0-res.json ├── hendrycksTest-econometrics-v0-loglikelihood ├── hendrycksTest-econometrics-v0-res.json ├── hendrycksTest-electrical_engineering-v0-loglikelihood ├── hendrycksTest-electrical_engineering-v0-res.json ├── hendrycksTest-elementary_mathematics-v0-loglikelihood ├── hendrycksTest-elementary_mathematics-v0-res.json ├── hendrycksTest-formal_logic-v0-loglikelihood ├── hendrycksTest-formal_logic-v0-res.json ├── hendrycksTest-global_facts-v0-loglikelihood ├── hendrycksTest-global_facts-v0-res.json ├── hendrycksTest-high_school_biology-v0-loglikelihood ├── hendrycksTest-high_school_biology-v0-res.json ├── hendrycksTest-high_school_chemistry-v0-loglikelihood ├── hendrycksTest-high_school_chemistry-v0-res.json ├── hendrycksTest-high_school_computer_science-v0-loglikelihood ├── hendrycksTest-high_school_computer_science-v0-res.json ├── hendrycksTest-high_school_european_history-v0-loglikelihood ├── hendrycksTest-high_school_european_history-v0-res.json ├── hendrycksTest-high_school_geography-v0-loglikelihood ├── hendrycksTest-high_school_geography-v0-res.json ├── hendrycksTest-high_school_government_and_politics-v0-loglikelihood ├── hendrycksTest-high_school_government_and_politics-v0-res.json ├── hendrycksTest-high_school_macroeconomics-v0-loglikelihood ├── hendrycksTest-high_school_macroeconomics-v0-res.json ├── hendrycksTest-high_school_mathematics-v0-loglikelihood ├── hendrycksTest-high_school_mathematics-v0-res.json ├── hendrycksTest-high_school_microeconomics-v0-loglikelihood ├── hendrycksTest-high_school_microeconomics-v0-res.json ├── hendrycksTest-high_school_physics-v0-loglikelihood ├── hendrycksTest-high_school_physics-v0-res.json ├── hendrycksTest-high_school_psychology-v0-loglikelihood ├── hendrycksTest-high_school_psychology-v0-res.json ├── hendrycksTest-high_school_statistics-v0-loglikelihood ├── hendrycksTest-high_school_statistics-v0-res.json ├── hendrycksTest-high_school_us_history-v0-loglikelihood ├── hendrycksTest-high_school_us_history-v0-res.json ├── hendrycksTest-high_school_world_history-v0-loglikelihood ├── hendrycksTest-high_school_world_history-v0-res.json ├── hendrycksTest-human_aging-v0-loglikelihood ├── hendrycksTest-human_aging-v0-res.json ├── hendrycksTest-human_sexuality-v0-loglikelihood ├── hendrycksTest-human_sexuality-v0-res.json ├── hendrycksTest-international_law-v0-loglikelihood ├── hendrycksTest-international_law-v0-res.json ├── hendrycksTest-jurisprudence-v0-loglikelihood ├── hendrycksTest-jurisprudence-v0-res.json ├── hendrycksTest-logical_fallacies-v0-loglikelihood ├── hendrycksTest-logical_fallacies-v0-res.json ├── hendrycksTest-machine_learning-v0-loglikelihood ├── hendrycksTest-machine_learning-v0-res.json ├── hendrycksTest-management-v0-loglikelihood ├── hendrycksTest-management-v0-res.json ├── hendrycksTest-marketing-v0-loglikelihood ├── hendrycksTest-marketing-v0-res.json ├── hendrycksTest-medical_genetics-v0-loglikelihood ├── hendrycksTest-medical_genetics-v0-res.json ├── hendrycksTest-miscellaneous-v0-loglikelihood ├── hendrycksTest-miscellaneous-v0-res.json ├── hendrycksTest-moral_disputes-v0-loglikelihood ├── hendrycksTest-moral_disputes-v0-res.json ├── hendrycksTest-moral_scenarios-v0-loglikelihood ├── hendrycksTest-moral_scenarios-v0-res.json ├── hendrycksTest-nutrition-v0-loglikelihood ├── hendrycksTest-nutrition-v0-res.json ├── hendrycksTest-philosophy-v0-loglikelihood ├── hendrycksTest-philosophy-v0-res.json ├── hendrycksTest-prehistory-v0-loglikelihood ├── hendrycksTest-prehistory-v0-res.json ├── hendrycksTest-professional_accounting-v0-loglikelihood ├── hendrycksTest-professional_accounting-v0-res.json ├── hendrycksTest-professional_law-v0-loglikelihood ├── hendrycksTest-professional_law-v0-res.json ├── hendrycksTest-professional_medicine-v0-loglikelihood ├── hendrycksTest-professional_medicine-v0-res.json ├── hendrycksTest-professional_psychology-v0-loglikelihood ├── hendrycksTest-professional_psychology-v0-res.json ├── hendrycksTest-public_relations-v0-loglikelihood ├── hendrycksTest-public_relations-v0-res.json ├── hendrycksTest-security_studies-v0-loglikelihood ├── hendrycksTest-security_studies-v0-res.json ├── hendrycksTest-sociology-v0-loglikelihood ├── hendrycksTest-sociology-v0-res.json ├── hendrycksTest-us_foreign_policy-v0-loglikelihood ├── hendrycksTest-us_foreign_policy-v0-res.json ├── hendrycksTest-virology-v0-loglikelihood ├── hendrycksTest-virology-v0-res.json ├── hendrycksTest-world_religions-v0-loglikelihood ├── hendrycksTest-world_religions-v0-res.json ├── iwslt17-ar-en-v0-greedy_until ├── iwslt17-ar-en-v0-res.json ├── iwslt17-en-ar-v0-greedy_until ├── iwslt17-en-ar-v0-res.json ├── lambada-v0-loglikelihood ├── lambada-v0-res.json ├── lambada_cloze-v0-loglikelihood ├── lambada_cloze-v0-res.json ├── lambada_mt_de-v0-loglikelihood ├── lambada_mt_de-v0-res.json ├── lambada_mt_en-v0-loglikelihood ├── lambada_mt_en-v0-res.json ├── lambada_mt_es-v0-loglikelihood ├── lambada_mt_es-v0-res.json ├── lambada_mt_fr-v0-loglikelihood ├── lambada_mt_fr-v0-res.json ├── lambada_mt_it-v0-loglikelihood ├── lambada_mt_it-v0-res.json ├── lambada_openai-v0-loglikelihood ├── lambada_openai-v0-res.json ├── lambada_openai_cloze-v0-loglikelihood ├── lambada_openai_cloze-v0-res.json ├── lambada_openai_mt_de-v0-loglikelihood ├── lambada_openai_mt_de-v0-res.json ├── lambada_openai_mt_en-v0-loglikelihood ├── lambada_openai_mt_en-v0-res.json ├── lambada_openai_mt_es-v0-loglikelihood ├── lambada_openai_mt_es-v0-res.json ├── lambada_openai_mt_fr-v0-loglikelihood ├── lambada_openai_mt_fr-v0-res.json ├── lambada_openai_mt_it-v0-loglikelihood ├── lambada_openai_mt_it-v0-res.json ├── lambada_standard-v0-loglikelihood ├── lambada_standard-v0-res.json ├── lambada_standard_cloze-v0-loglikelihood ├── lambada_standard_cloze-v0-res.json ├── logiqa-v0-loglikelihood ├── logiqa-v0-res.json ├── math_algebra-v0-greedy_until ├── math_algebra-v0-res.json ├── math_algebra-v1-greedy_until ├── math_algebra-v1-res.json ├── math_counting_and_prob-v0-greedy_until ├── math_counting_and_prob-v0-res.json ├── math_counting_and_prob-v1-greedy_until ├── math_counting_and_prob-v1-res.json ├── math_geometry-v0-greedy_until ├── math_geometry-v0-res.json ├── math_geometry-v1-greedy_until ├── math_geometry-v1-res.json ├── math_intermediate_algebra-v0-greedy_until ├── math_intermediate_algebra-v0-res.json ├── math_intermediate_algebra-v1-greedy_until ├── math_intermediate_algebra-v1-res.json ├── math_num_theory-v0-greedy_until ├── math_num_theory-v0-res.json ├── math_num_theory-v1-greedy_until ├── math_num_theory-v1-res.json ├── math_prealgebra-v0-greedy_until ├── math_prealgebra-v0-res.json ├── math_prealgebra-v1-greedy_until ├── math_prealgebra-v1-res.json ├── math_precalc-v0-greedy_until ├── math_precalc-v0-res.json ├── math_precalc-v1-greedy_until ├── math_precalc-v1-res.json ├── mathqa-v0-loglikelihood ├── mathqa-v0-res.json ├── mc_taco-v0-loglikelihood ├── mc_taco-v0-res.json ├── mnli-v0-loglikelihood ├── mnli-v0-res.json ├── mnli_mismatched-v0-loglikelihood ├── mnli_mismatched-v0-res.json ├── mrpc-v0-loglikelihood ├── mrpc-v0-res.json ├── multirc-v0-loglikelihood ├── multirc-v0-res.json ├── multirc-v1-loglikelihood ├── multirc-v1-res.json ├── mutual-v0-loglikelihood ├── mutual-v0-res.json ├── mutual-v1-loglikelihood ├── mutual-v1-res.json ├── mutual_plus-v0-loglikelihood ├── mutual_plus-v0-res.json ├── mutual_plus-v1-loglikelihood ├── mutual_plus-v1-res.json ├── openbookqa-v0-loglikelihood ├── openbookqa-v0-res.json ├── pile_arxiv-v0-loglikelihood_rolling ├── pile_arxiv-v0-res.json ├── pile_arxiv-v1-loglikelihood_rolling ├── pile_arxiv-v1-res.json ├── pile_bookcorpus2-v0-loglikelihood_rolling ├── pile_bookcorpus2-v0-res.json ├── pile_bookcorpus2-v1-loglikelihood_rolling ├── pile_bookcorpus2-v1-res.json ├── pile_books3-v0-loglikelihood_rolling ├── pile_books3-v0-res.json ├── pile_books3-v1-loglikelihood_rolling ├── pile_books3-v1-res.json ├── pile_dm-mathematics-v0-loglikelihood_rolling ├── pile_dm-mathematics-v0-res.json ├── pile_dm-mathematics-v1-loglikelihood_rolling ├── pile_dm-mathematics-v1-res.json ├── pile_enron-v0-loglikelihood_rolling ├── pile_enron-v0-res.json ├── pile_enron-v1-loglikelihood_rolling ├── pile_enron-v1-res.json ├── pile_europarl-v0-loglikelihood_rolling ├── pile_europarl-v0-res.json ├── pile_europarl-v1-loglikelihood_rolling ├── pile_europarl-v1-res.json ├── pile_freelaw-v0-loglikelihood_rolling ├── pile_freelaw-v0-res.json ├── pile_freelaw-v1-loglikelihood_rolling ├── pile_freelaw-v1-res.json ├── pile_github-v0-loglikelihood_rolling ├── pile_github-v0-res.json ├── pile_github-v1-loglikelihood_rolling ├── pile_github-v1-res.json ├── pile_gutenberg-v0-loglikelihood_rolling ├── pile_gutenberg-v0-res.json ├── pile_gutenberg-v1-loglikelihood_rolling ├── pile_gutenberg-v1-res.json ├── pile_hackernews-v0-loglikelihood_rolling ├── pile_hackernews-v0-res.json ├── pile_hackernews-v1-loglikelihood_rolling ├── pile_hackernews-v1-res.json ├── pile_nih-exporter-v0-loglikelihood_rolling ├── pile_nih-exporter-v0-res.json ├── pile_nih-exporter-v1-loglikelihood_rolling ├── pile_nih-exporter-v1-res.json ├── pile_opensubtitles-v0-loglikelihood_rolling ├── pile_opensubtitles-v0-res.json ├── pile_opensubtitles-v1-loglikelihood_rolling ├── pile_opensubtitles-v1-res.json ├── pile_openwebtext2-v0-loglikelihood_rolling ├── pile_openwebtext2-v0-res.json ├── pile_openwebtext2-v1-loglikelihood_rolling ├── pile_openwebtext2-v1-res.json ├── pile_philpapers-v0-loglikelihood_rolling ├── pile_philpapers-v0-res.json ├── pile_philpapers-v1-loglikelihood_rolling ├── pile_philpapers-v1-res.json ├── pile_pile-cc-v0-loglikelihood_rolling ├── pile_pile-cc-v0-res.json ├── pile_pile-cc-v1-loglikelihood_rolling ├── pile_pile-cc-v1-res.json ├── pile_pubmed-abstracts-v0-loglikelihood_rolling ├── pile_pubmed-abstracts-v0-res.json ├── pile_pubmed-abstracts-v1-loglikelihood_rolling ├── pile_pubmed-abstracts-v1-res.json ├── pile_pubmed-central-v0-loglikelihood_rolling ├── pile_pubmed-central-v0-res.json ├── pile_pubmed-central-v1-loglikelihood_rolling ├── pile_pubmed-central-v1-res.json ├── pile_stackexchange-v0-loglikelihood_rolling ├── pile_stackexchange-v0-res.json ├── pile_stackexchange-v1-loglikelihood_rolling ├── pile_stackexchange-v1-res.json ├── pile_ubuntu-irc-v0-loglikelihood_rolling ├── pile_ubuntu-irc-v0-res.json ├── pile_ubuntu-irc-v1-loglikelihood_rolling ├── pile_ubuntu-irc-v1-res.json ├── pile_uspto-v0-loglikelihood_rolling ├── pile_uspto-v0-res.json ├── pile_uspto-v1-loglikelihood_rolling ├── pile_uspto-v1-res.json ├── pile_wikipedia-v0-loglikelihood_rolling ├── pile_wikipedia-v0-res.json ├── pile_wikipedia-v1-loglikelihood_rolling ├── pile_wikipedia-v1-res.json ├── pile_youtubesubtitles-v0-loglikelihood_rolling ├── pile_youtubesubtitles-v0-res.json ├── pile_youtubesubtitles-v1-loglikelihood_rolling ├── pile_youtubesubtitles-v1-res.json ├── piqa-v0-loglikelihood ├── piqa-v0-res.json ├── prost-v0-loglikelihood ├── prost-v0-res.json ├── pubmedqa-v0-loglikelihood ├── pubmedqa-v0-res.json ├── qa4mre_2011-v0-loglikelihood ├── qa4mre_2011-v0-res.json ├── qa4mre_2012-v0-loglikelihood ├── qa4mre_2012-v0-res.json ├── qa4mre_2013-v0-loglikelihood ├── qa4mre_2013-v0-res.json ├── qnli-v0-loglikelihood ├── qnli-v0-res.json ├── qqp-v0-loglikelihood ├── qqp-v0-res.json ├── race-v0-loglikelihood ├── race-v0-res.json ├── random_insertion-v0-greedy_until ├── random_insertion-v0-res.json ├── record-v0-loglikelihood ├── record-v0-res.json ├── reversed_words-v0-greedy_until ├── reversed_words-v0-res.json ├── rte-v0-loglikelihood ├── rte-v0-res.json ├── sciq-v0-loglikelihood ├── sciq-v0-res.json ├── squad2-v0-greedy_until ├── squad2-v0-loglikelihood ├── squad2-v0-res.json ├── squad2-v1-greedy_until ├── squad2-v1-loglikelihood ├── squad2-v1-res.json ├── sst-v0-loglikelihood ├── sst-v0-res.json ├── swag-v0-loglikelihood ├── swag-v0-res.json ├── textsynth_test_0a89c2739f9598b4be2674b0a8e43931d7f3f0b696970bcba31f9b52bdf12297.pkl ├── textsynth_test_0c1c14571add7903b89e588c8212572b95bb57b334fc0752c89a7e045a5f63ae.pkl ├── textsynth_test_3092d07756f3e1d010c07524cc8a2ecba7f0c19f9e39f2aaf2bf440bfe328004.pkl ├── textsynth_test_434076260b6af3a46b7a5eaceec3306a5872c400a3872f744280b237455a0f8e.pkl ├── textsynth_test_49c47ae40e11f349f2f6b492128188b1b2bc103a421c676ee4b2142a68b43516.pkl ├── textsynth_test_4fd8d66a6dad7f602b40e5d7dc298d6fe329299d086a4659743a41f4a4012659.pkl ├── textsynth_test_51b5302f157cf224f694ccad973f255ae19e9e061d533256bdf75b04e0a917ab.pkl ├── textsynth_test_6d6c62dd70caaa208712bf766deaf419cfac89538d4ab7745621e339394c0c23.pkl ├── textsynth_test_7209c4617547bfe17cb9e7f5f735fe35822d650aefdc5fbeeaf0c1724effbe09.pkl ├── textsynth_test_7afdc285388e51094e12645f305328c759574fa3ec9751631025f8ad5ebf9f3e.pkl ├── textsynth_test_9d5f33dbfe1e254928c89f5ed85e4c010d888065f55a8f1b863bc1eb0340a5f2.pkl ├── textsynth_test_abcbcba648d89e5d81a50511a6d24ddeb538de2ffe108c1370dd74ce6ac8038d.pkl ├── textsynth_test_b1cbb29666cce5e31a1e97695858137398a0885ca5d5d98f515404fb6aeb99e7.pkl ├── textsynth_test_e7ad1e9f52a39e1ddd1e50f3c57ffa4546728dd150a67c0a0ddc8675c04e15d1.pkl ├── textsynth_test_f4bfe4beb605bd52a8ab6be3c9293639e7e2261d98de58159d15ccb83131bf4e.pkl ├── toxigen-v0-loglikelihood ├── toxigen-v0-res.json ├── triviaqa-v0-loglikelihood ├── triviaqa-v0-res.json ├── triviaqa-v1-loglikelihood ├── triviaqa-v1-res.json ├── truthfulqa_gen-v0-greedy_until ├── truthfulqa_gen-v0-res.json ├── truthfulqa_gen-v1-greedy_until ├── truthfulqa_gen-v1-res.json ├── truthfulqa_mc-v0-loglikelihood ├── truthfulqa_mc-v0-res.json ├── truthfulqa_mc-v1-loglikelihood ├── truthfulqa_mc-v1-res.json ├── webqs-v0-loglikelihood ├── webqs-v0-res.json ├── wic-v0-loglikelihood ├── wic-v0-res.json ├── wikitext-v0-loglikelihood_rolling ├── wikitext-v0-res.json ├── wikitext-v1-loglikelihood_rolling ├── wikitext-v1-res.json ├── winogrande-v0-loglikelihood ├── winogrande-v0-res.json ├── wmt14-en-fr-v0-greedy_until ├── wmt14-en-fr-v0-res.json ├── wmt14-fr-en-v0-greedy_until ├── wmt14-fr-en-v0-res.json ├── wmt16-de-en-v0-greedy_until ├── wmt16-de-en-v0-res.json ├── wmt16-en-de-v0-greedy_until ├── wmt16-en-de-v0-res.json ├── wmt16-en-ro-v0-greedy_until ├── wmt16-en-ro-v0-res.json ├── wmt16-ro-en-v0-greedy_until ├── wmt16-ro-en-v0-res.json ├── wmt20-cs-en-v0-greedy_until ├── wmt20-cs-en-v0-res.json ├── wmt20-de-en-v0-greedy_until ├── wmt20-de-en-v0-res.json ├── wmt20-de-fr-v0-greedy_until ├── wmt20-de-fr-v0-res.json ├── wmt20-en-cs-v0-greedy_until ├── wmt20-en-cs-v0-res.json ├── wmt20-en-de-v0-greedy_until ├── wmt20-en-de-v0-res.json ├── wmt20-en-iu-v0-greedy_until ├── wmt20-en-iu-v0-res.json ├── wmt20-en-ja-v0-greedy_until ├── wmt20-en-ja-v0-res.json ├── wmt20-en-ja-v1-greedy_until ├── wmt20-en-ja-v1-res.json ├── wmt20-en-km-v0-greedy_until ├── wmt20-en-km-v0-res.json ├── wmt20-en-pl-v0-greedy_until ├── wmt20-en-pl-v0-res.json ├── wmt20-en-ps-v0-greedy_until ├── wmt20-en-ps-v0-res.json ├── wmt20-en-ru-v0-greedy_until ├── wmt20-en-ru-v0-res.json ├── wmt20-en-ta-v0-greedy_until ├── wmt20-en-ta-v0-res.json ├── wmt20-en-zh-v0-greedy_until ├── wmt20-en-zh-v0-res.json ├── wmt20-en-zh-v1-greedy_until ├── wmt20-en-zh-v1-res.json ├── wmt20-fr-de-v0-greedy_until ├── wmt20-fr-de-v0-res.json ├── wmt20-iu-en-v0-greedy_until ├── wmt20-iu-en-v0-res.json ├── wmt20-ja-en-v0-greedy_until ├── wmt20-ja-en-v0-res.json ├── wmt20-km-en-v0-greedy_until ├── wmt20-km-en-v0-res.json ├── wmt20-pl-en-v0-greedy_until ├── wmt20-pl-en-v0-res.json ├── wmt20-ps-en-v0-greedy_until ├── wmt20-ps-en-v0-res.json ├── wmt20-ru-en-v0-greedy_until ├── wmt20-ru-en-v0-res.json ├── wmt20-ta-en-v0-greedy_until ├── wmt20-ta-en-v0-res.json ├── wmt20-zh-en-v0-greedy_until ├── wmt20-zh-en-v0-res.json ├── wnli-v0-loglikelihood ├── wnli-v0-res.json ├── wnli-v1-loglikelihood ├── wnli-v1-res.json ├── wsc-v0-loglikelihood ├── wsc-v0-res.json ├── wsc273-v0-loglikelihood └── wsc273-v0-res.json /.coveragerc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/.coveragerc -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/.flake8 -------------------------------------------------------------------------------- /.github/workflows/pull_request.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/.github/workflows/pull_request.yml -------------------------------------------------------------------------------- /.github/workflows/python-app.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/.github/workflows/python-app.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /CITATION.bib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/CITATION.bib -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @jon-tow @StellaAthena 2 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/LICENSE.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/README.md -------------------------------------------------------------------------------- /docs/decontamination.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/docs/decontamination.md -------------------------------------------------------------------------------- /docs/description_guide.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/docs/description_guide.md -------------------------------------------------------------------------------- /docs/img/fewshot_example_gpt3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/docs/img/fewshot_example_gpt3.png -------------------------------------------------------------------------------- /docs/jptasks.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/docs/jptasks.md -------------------------------------------------------------------------------- /docs/prompt_templates.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/docs/prompt_templates.md -------------------------------------------------------------------------------- /docs/task_guide.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/docs/task_guide.md -------------------------------------------------------------------------------- /docs/task_table.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/docs/task_table.md -------------------------------------------------------------------------------- /ignore.txt: -------------------------------------------------------------------------------- 1 | ROUGE 2 | rouge 3 | nin 4 | -------------------------------------------------------------------------------- /lm_eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/base.py -------------------------------------------------------------------------------- /lm_eval/datasets/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/README.md -------------------------------------------------------------------------------- /lm_eval/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/asdiv/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/asdiv/asdiv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/asdiv/asdiv.py -------------------------------------------------------------------------------- /lm_eval/datasets/asdiv/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/asdiv/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/coqa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/coqa/coqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/coqa/coqa.py -------------------------------------------------------------------------------- /lm_eval/datasets/coqa/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/coqa/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/drop/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/drop/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/drop/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/drop/drop.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/drop/drop.py -------------------------------------------------------------------------------- /lm_eval/datasets/headqa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/headqa/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/headqa/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/headqa/headqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/headqa/headqa.py -------------------------------------------------------------------------------- /lm_eval/datasets/hendrycks_ethics/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/hendrycks_ethics/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/hendrycks_ethics/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/hendrycks_ethics/hendrycks_ethics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/hendrycks_ethics/hendrycks_ethics.py -------------------------------------------------------------------------------- /lm_eval/datasets/hendrycks_math/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/hendrycks_math/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/hendrycks_math/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/hendrycks_math/hendrycks_math.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/hendrycks_math/hendrycks_math.py -------------------------------------------------------------------------------- /lm_eval/datasets/lambada_ja/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/lambada_ja/lambada_ja.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/lambada_ja/lambada_ja.py -------------------------------------------------------------------------------- /lm_eval/datasets/logiqa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/logiqa/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/logiqa/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/logiqa/logiqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/logiqa/logiqa.py -------------------------------------------------------------------------------- /lm_eval/datasets/mutual/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/mutual/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/mutual/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/mutual/mutual.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/mutual/mutual.py -------------------------------------------------------------------------------- /lm_eval/datasets/pile/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/pile/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/pile/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/pile/pile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/pile/pile.py -------------------------------------------------------------------------------- /lm_eval/datasets/quac/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/quac/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/quac/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/quac/quac.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/quac/quac.py -------------------------------------------------------------------------------- /lm_eval/datasets/sat_analogies/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/sat_analogies/sat_analogies.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/sat_analogies/sat_analogies.py -------------------------------------------------------------------------------- /lm_eval/datasets/triviaqa/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/triviaqa/README.md -------------------------------------------------------------------------------- /lm_eval/datasets/triviaqa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/triviaqa/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/triviaqa/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/triviaqa/triviaqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/triviaqa/triviaqa.py -------------------------------------------------------------------------------- /lm_eval/datasets/unscramble/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/datasets/unscramble/dataset_infos.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/unscramble/dataset_infos.json -------------------------------------------------------------------------------- /lm_eval/datasets/unscramble/unscramble.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/datasets/unscramble/unscramble.py -------------------------------------------------------------------------------- /lm_eval/decontamination/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/decontamination/archiver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/decontamination/archiver.py -------------------------------------------------------------------------------- /lm_eval/decontamination/decontaminate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/decontamination/decontaminate.py -------------------------------------------------------------------------------- /lm_eval/decontamination/janitor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/decontamination/janitor.py -------------------------------------------------------------------------------- /lm_eval/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/evaluator.py -------------------------------------------------------------------------------- /lm_eval/jasquad/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/jasquad/README.md -------------------------------------------------------------------------------- /lm_eval/jasquad/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/jasquad/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/jasquad/evaluate.py -------------------------------------------------------------------------------- /lm_eval/jasquad/jasquad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/jasquad/jasquad.py -------------------------------------------------------------------------------- /lm_eval/jasquad/requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} 2 | -------------------------------------------------------------------------------- /lm_eval/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/metrics.py -------------------------------------------------------------------------------- /lm_eval/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/models/__init__.py -------------------------------------------------------------------------------- /lm_eval/models/dummy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/models/dummy.py -------------------------------------------------------------------------------- /lm_eval/models/gpt2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/models/gpt2.py -------------------------------------------------------------------------------- /lm_eval/models/gpt3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/models/gpt3.py -------------------------------------------------------------------------------- /lm_eval/models/huggingface.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/models/huggingface.py -------------------------------------------------------------------------------- /lm_eval/models/textsynth.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/models/textsynth.py -------------------------------------------------------------------------------- /lm_eval/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/prompts.py -------------------------------------------------------------------------------- /lm_eval/suites/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/suites/__init__.py -------------------------------------------------------------------------------- /lm_eval/suites/configs/ja8.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/suites/configs/ja8.conf -------------------------------------------------------------------------------- /lm_eval/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/__init__.py -------------------------------------------------------------------------------- /lm_eval/tasks/anli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/anli.py -------------------------------------------------------------------------------- /lm_eval/tasks/arc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/arc.py -------------------------------------------------------------------------------- /lm_eval/tasks/arithmetic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/arithmetic.py -------------------------------------------------------------------------------- /lm_eval/tasks/asdiv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/asdiv.py -------------------------------------------------------------------------------- /lm_eval/tasks/blimp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/blimp.py -------------------------------------------------------------------------------- /lm_eval/tasks/cbt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/cbt.py -------------------------------------------------------------------------------- /lm_eval/tasks/coqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/coqa.py -------------------------------------------------------------------------------- /lm_eval/tasks/crowspairs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/crowspairs.py -------------------------------------------------------------------------------- /lm_eval/tasks/drop.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/drop.py -------------------------------------------------------------------------------- /lm_eval/tasks/glue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/glue.py -------------------------------------------------------------------------------- /lm_eval/tasks/gsm8k.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/gsm8k.py -------------------------------------------------------------------------------- /lm_eval/tasks/headqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/headqa.py -------------------------------------------------------------------------------- /lm_eval/tasks/hellaswag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/hellaswag.py -------------------------------------------------------------------------------- /lm_eval/tasks/hendrycks_ethics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/hendrycks_ethics.py -------------------------------------------------------------------------------- /lm_eval/tasks/hendrycks_math.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/hendrycks_math.py -------------------------------------------------------------------------------- /lm_eval/tasks/hendrycks_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/hendrycks_test.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/__init__.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/jaqket_v1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/jaqket_v1.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/jaqket_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/jaqket_v2.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/jaquad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/jaquad.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/jblimp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/jblimp.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/jcola.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/jcola.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/jcommonsenseqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/jcommonsenseqa.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/jnli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/jnli.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/jsquad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/jsquad.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/marc_ja.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/marc_ja.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/mgsm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/mgsm.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/wikilingua_ja.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/wikilingua_ja.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/xlsum_ja.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/xlsum_ja.py -------------------------------------------------------------------------------- /lm_eval/tasks/ja/xwinograd_ja.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/ja/xwinograd_ja.py -------------------------------------------------------------------------------- /lm_eval/tasks/lambada.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/lambada.py -------------------------------------------------------------------------------- /lm_eval/tasks/lambada_cloze.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/lambada_cloze.py -------------------------------------------------------------------------------- /lm_eval/tasks/lambada_multilingual.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/lambada_multilingual.py -------------------------------------------------------------------------------- /lm_eval/tasks/logiqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/logiqa.py -------------------------------------------------------------------------------- /lm_eval/tasks/mathqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/mathqa.py -------------------------------------------------------------------------------- /lm_eval/tasks/mc_taco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/mc_taco.py -------------------------------------------------------------------------------- /lm_eval/tasks/mutual.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/mutual.py -------------------------------------------------------------------------------- /lm_eval/tasks/naturalqs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/naturalqs.py -------------------------------------------------------------------------------- /lm_eval/tasks/openbookqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/openbookqa.py -------------------------------------------------------------------------------- /lm_eval/tasks/pile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/pile.py -------------------------------------------------------------------------------- /lm_eval/tasks/piqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/piqa.py -------------------------------------------------------------------------------- /lm_eval/tasks/prost.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/prost.py -------------------------------------------------------------------------------- /lm_eval/tasks/pubmedqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/pubmedqa.py -------------------------------------------------------------------------------- /lm_eval/tasks/qa4mre.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/qa4mre.py -------------------------------------------------------------------------------- /lm_eval/tasks/qasper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/qasper.py -------------------------------------------------------------------------------- /lm_eval/tasks/quac.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/quac.py -------------------------------------------------------------------------------- /lm_eval/tasks/race.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/race.py -------------------------------------------------------------------------------- /lm_eval/tasks/sat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/sat.py -------------------------------------------------------------------------------- /lm_eval/tasks/sciq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/sciq.py -------------------------------------------------------------------------------- /lm_eval/tasks/squad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/squad.py -------------------------------------------------------------------------------- /lm_eval/tasks/storycloze.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/storycloze.py -------------------------------------------------------------------------------- /lm_eval/tasks/superglue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/superglue.py -------------------------------------------------------------------------------- /lm_eval/tasks/swag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/swag.py -------------------------------------------------------------------------------- /lm_eval/tasks/toxigen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/toxigen.py -------------------------------------------------------------------------------- /lm_eval/tasks/translation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/translation.py -------------------------------------------------------------------------------- /lm_eval/tasks/triviaqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/triviaqa.py -------------------------------------------------------------------------------- /lm_eval/tasks/truthfulqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/truthfulqa.py -------------------------------------------------------------------------------- /lm_eval/tasks/unscramble.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/unscramble.py -------------------------------------------------------------------------------- /lm_eval/tasks/webqs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/webqs.py -------------------------------------------------------------------------------- /lm_eval/tasks/wikitext.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/wikitext.py -------------------------------------------------------------------------------- /lm_eval/tasks/winogrande.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/winogrande.py -------------------------------------------------------------------------------- /lm_eval/tasks/wsc273.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/tasks/wsc273.py -------------------------------------------------------------------------------- /lm_eval/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/lm_eval/utils.py -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | scripts/main_eval.py -------------------------------------------------------------------------------- /models/abeja-gpt-neox-japanese-2.7b/harness.jsquad-1.2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/abeja-gpt-neox-japanese-2.7b/harness.jsquad-1.2.sh -------------------------------------------------------------------------------- /models/abeja-gpt-neox-japanese-2.7b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/abeja-gpt-neox-japanese-2.7b/harness.sh -------------------------------------------------------------------------------- /models/abeja-gpt-neox-japanese-2.7b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/abeja-gpt-neox-japanese-2.7b/result.json -------------------------------------------------------------------------------- /models/abeja-gpt-neox-japanese-2.7b/result.jsquad-1.2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/abeja-gpt-neox-japanese-2.7b/result.jsquad-1.2.json -------------------------------------------------------------------------------- /models/community/rinna-instruct-1b_0.1.0/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/community/rinna-instruct-1b_0.1.0/harness.sh -------------------------------------------------------------------------------- /models/community/rinna-instruct-1b_0.1.0/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/community/rinna-instruct-1b_0.1.0/result.json -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-1b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-1b/harness.sh -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-1b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-1b/result.json -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-1b/result.mgsm.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-1b/result.mgsm.json -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-3b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-3b/harness.sh -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-3b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-3b/result.json -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-3b/result.mgsm.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-3b/result.mgsm.json -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-7b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-7b/harness.sh -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-7b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-7b/result.json -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-7b/result.mgsm.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-7b/result.mgsm.json -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-large/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-large/harness.sh -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-large/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-large/result.json -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-medium/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-medium/harness.sh -------------------------------------------------------------------------------- /models/cyberagent/cyberagent-open-calm-medium/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/cyberagent/cyberagent-open-calm-medium/result.json -------------------------------------------------------------------------------- /models/harness.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/harness.conf -------------------------------------------------------------------------------- /models/llama/llama-13b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama/llama-13b/harness.sh -------------------------------------------------------------------------------- /models/llama/llama-13b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama/llama-13b/result.json -------------------------------------------------------------------------------- /models/llama/llama-30b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama/llama-30b/harness.sh -------------------------------------------------------------------------------- /models/llama/llama-30b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama/llama-30b/result.json -------------------------------------------------------------------------------- /models/llama/llama-65b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama/llama-65b/harness.sh -------------------------------------------------------------------------------- /models/llama/llama-65b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama/llama-65b/result.json -------------------------------------------------------------------------------- /models/llama/llama-7b/harness.jsquad-1.2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama/llama-7b/harness.jsquad-1.2.sh -------------------------------------------------------------------------------- /models/llama/llama-7b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama/llama-7b/harness.sh -------------------------------------------------------------------------------- /models/llama/llama-7b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama/llama-7b/result.json -------------------------------------------------------------------------------- /models/llama/llama-7b/result.jsquad-1.2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama/llama-7b/result.jsquad-1.2.json -------------------------------------------------------------------------------- /models/llama2/llama2-13b-chat/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-13b-chat/harness.sh -------------------------------------------------------------------------------- /models/llama2/llama2-13b-chat/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-13b-chat/result.json -------------------------------------------------------------------------------- /models/llama2/llama2-13b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-13b/harness.sh -------------------------------------------------------------------------------- /models/llama2/llama2-13b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-13b/result.json -------------------------------------------------------------------------------- /models/llama2/llama2-7b-chat/harness.jsquad-1.2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-7b-chat/harness.jsquad-1.2.sh -------------------------------------------------------------------------------- /models/llama2/llama2-7b-chat/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-7b-chat/harness.sh -------------------------------------------------------------------------------- /models/llama2/llama2-7b-chat/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-7b-chat/result.json -------------------------------------------------------------------------------- /models/llama2/llama2-7b-chat/result.jsquad-1.2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-7b-chat/result.jsquad-1.2.json -------------------------------------------------------------------------------- /models/llama2/llama2-7b/harness.jsquad-1.2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-7b/harness.jsquad-1.2.sh -------------------------------------------------------------------------------- /models/llama2/llama2-7b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-7b/harness.sh -------------------------------------------------------------------------------- /models/llama2/llama2-7b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-7b/result.json -------------------------------------------------------------------------------- /models/llama2/llama2-7b/result.jsquad-1.2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/llama2/llama2-7b/result.jsquad-1.2.json -------------------------------------------------------------------------------- /models/openai/gpt3/result.mgsm.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/openai/gpt3/result.mgsm.json -------------------------------------------------------------------------------- /models/rinna/harness.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/harness.conf -------------------------------------------------------------------------------- /models/rinna/rinna-bilingual-gpt-neox-4b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/rinna-bilingual-gpt-neox-4b/harness.sh -------------------------------------------------------------------------------- /models/rinna/rinna-bilingual-gpt-neox-4b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/rinna-bilingual-gpt-neox-4b/result.json -------------------------------------------------------------------------------- /models/rinna/rinna-japanese-gpt-1b/harness.jsquad-1.2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/rinna-japanese-gpt-1b/harness.jsquad-1.2.sh -------------------------------------------------------------------------------- /models/rinna/rinna-japanese-gpt-1b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/rinna-japanese-gpt-1b/harness.sh -------------------------------------------------------------------------------- /models/rinna/rinna-japanese-gpt-1b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/rinna-japanese-gpt-1b/result.json -------------------------------------------------------------------------------- /models/rinna/rinna-japanese-gpt-1b/result.jsquad-1.2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/rinna-japanese-gpt-1b/result.jsquad-1.2.json -------------------------------------------------------------------------------- /models/rinna/rinna-japanese-gpt-neox-3.6b/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/rinna-japanese-gpt-neox-3.6b/harness.sh -------------------------------------------------------------------------------- /models/rinna/rinna-japanese-gpt-neox-3.6b/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/rinna-japanese-gpt-neox-3.6b/result.json -------------------------------------------------------------------------------- /models/rinna/rinna-japanese-gpt-neox-3.6b/result.mgsm.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/rinna-japanese-gpt-neox-3.6b/result.mgsm.json -------------------------------------------------------------------------------- /models/rinna/rinna-japanese-gpt-neox-small/harness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/rinna-japanese-gpt-neox-small/harness.sh -------------------------------------------------------------------------------- /models/rinna/rinna-japanese-gpt-neox-small/result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/rinna/rinna-japanese-gpt-neox-small/result.json -------------------------------------------------------------------------------- /models/stablelm/harness.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/stablelm/harness.conf -------------------------------------------------------------------------------- /models/stablelm/stablelm-jp-3b-ja50_rp50-700b/harness.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/models/stablelm/stablelm-jp-3b-ja50_rp50-700b/harness.conf -------------------------------------------------------------------------------- /pile_statistics.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/pile_statistics.json -------------------------------------------------------------------------------- /requirements-ja.txt: -------------------------------------------------------------------------------- 1 | emoji 2 | fugashi==1.2.1 3 | neologdn>=0.5.2 4 | unidic-lite==1.0.8 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/clean_training_data/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/clean_training_data/README.md -------------------------------------------------------------------------------- /scripts/clean_training_data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/clean_training_data/compress_and_package.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/clean_training_data/compress_and_package.py -------------------------------------------------------------------------------- /scripts/clean_training_data/generate_13_grams.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/clean_training_data/generate_13_grams.py -------------------------------------------------------------------------------- /scripts/clean_training_data/investigate_pile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/clean_training_data/investigate_pile.py -------------------------------------------------------------------------------- /scripts/clean_training_data/janitor_util.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/clean_training_data/janitor_util.cpp -------------------------------------------------------------------------------- /scripts/clean_training_data/process_sorted_buckets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/clean_training_data/process_sorted_buckets.py -------------------------------------------------------------------------------- /scripts/clean_training_data/sort_13_gram_buckets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/clean_training_data/sort_13_gram_buckets.py -------------------------------------------------------------------------------- /scripts/compute_average_from_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/compute_average_from_json.py -------------------------------------------------------------------------------- /scripts/cost_estimate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/cost_estimate.py -------------------------------------------------------------------------------- /scripts/generate_harness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/generate_harness.py -------------------------------------------------------------------------------- /scripts/get_prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/get_prompts.py -------------------------------------------------------------------------------- /scripts/harness_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/harness_example.py -------------------------------------------------------------------------------- /scripts/main_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/main_eval.py -------------------------------------------------------------------------------- /scripts/make_gpt2_test_cases.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/make_gpt2_test_cases.py -------------------------------------------------------------------------------- /scripts/make_leaderboard.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/make_leaderboard.py -------------------------------------------------------------------------------- /scripts/make_table_tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/make_table_tasks.py -------------------------------------------------------------------------------- /scripts/merge_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/merge_json.py -------------------------------------------------------------------------------- /scripts/models.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/models.txt -------------------------------------------------------------------------------- /scripts/notify.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/notify.py -------------------------------------------------------------------------------- /scripts/run_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/run_eval.py -------------------------------------------------------------------------------- /scripts/run_suite.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/run_suite.py -------------------------------------------------------------------------------- /scripts/run_task.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/run_task.sh -------------------------------------------------------------------------------- /scripts/run_task_batch.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/run_task_batch.sh -------------------------------------------------------------------------------- /scripts/run_task_for_models.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/run_task_for_models.sh -------------------------------------------------------------------------------- /scripts/write_out.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/scripts/write_out.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/setup.py -------------------------------------------------------------------------------- /templates/new_multiple_choice_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/templates/new_multiple_choice_task.py -------------------------------------------------------------------------------- /templates/new_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/templates/new_task.py -------------------------------------------------------------------------------- /tests/test_description_dict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/test_description_dict.py -------------------------------------------------------------------------------- /tests/test_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/test_evaluator.py -------------------------------------------------------------------------------- /tests/test_generate_13_grams.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/test_generate_13_grams.py -------------------------------------------------------------------------------- /tests/test_janitor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/test_janitor.py -------------------------------------------------------------------------------- /tests/test_misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/test_misc.py -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/test_models.py -------------------------------------------------------------------------------- /tests/test_tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/test_tasks.py -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/test_utils.py -------------------------------------------------------------------------------- /tests/test_version_stable.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/test_version_stable.py -------------------------------------------------------------------------------- /tests/testdata/anagrams1-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/anagrams1-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/anagrams1-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/anagrams1-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/anagrams2-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/anagrams2-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/anagrams2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/anagrams2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/anli_r1-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/anli_r1-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/anli_r1-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/anli_r1-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/anli_r2-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/anli_r2-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/anli_r2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/anli_r2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/anli_r3-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/anli_r3-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/anli_r3-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/anli_r3-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arc_challenge-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arc_challenge-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arc_challenge-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arc_challenge-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arc_easy-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arc_easy-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arc_easy-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arc_easy-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arithmetic_1dc-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_1dc-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arithmetic_1dc-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_1dc-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arithmetic_2da-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_2da-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arithmetic_2da-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_2da-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arithmetic_2dm-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_2dm-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arithmetic_2dm-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_2dm-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arithmetic_2ds-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_2ds-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arithmetic_2ds-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_2ds-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arithmetic_3da-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_3da-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arithmetic_3da-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_3da-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arithmetic_3ds-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_3ds-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arithmetic_3ds-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_3ds-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arithmetic_4da-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_4da-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arithmetic_4da-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_4da-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arithmetic_4ds-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_4ds-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arithmetic_4ds-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_4ds-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arithmetic_5da-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_5da-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arithmetic_5da-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_5da-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/arithmetic_5ds-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_5ds-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/arithmetic_5ds-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/arithmetic_5ds-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_adjunct_island-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_adjunct_island-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_adjunct_island-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_adjunct_island-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_anaphor_gender_agreement-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_anaphor_gender_agreement-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_anaphor_number_agreement-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_anaphor_number_agreement-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_animate_subject_passive-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_animate_subject_passive-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_animate_subject_passive-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_animate_subject_passive-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_animate_subject_trans-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_animate_subject_trans-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_animate_subject_trans-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_animate_subject_trans-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_causative-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_causative-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_causative-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_causative-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_complex_NP_island-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_complex_NP_island-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_complex_NP_island-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_complex_NP_island-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_determiner_noun_agreement_1-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_determiner_noun_agreement_1-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_determiner_noun_agreement_2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_determiner_noun_agreement_2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_drop_argument-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_drop_argument-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_drop_argument-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_drop_argument-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_ellipsis_n_bar_1-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_ellipsis_n_bar_1-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_ellipsis_n_bar_1-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_ellipsis_n_bar_1-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_ellipsis_n_bar_2-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_ellipsis_n_bar_2-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_ellipsis_n_bar_2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_ellipsis_n_bar_2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_expletive_it_object_raising-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_expletive_it_object_raising-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_inchoative-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_inchoative-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_inchoative-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_inchoative-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_intransitive-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_intransitive-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_intransitive-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_intransitive-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_npi_present_1-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_npi_present_1-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_npi_present_1-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_npi_present_1-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_npi_present_2-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_npi_present_2-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_npi_present_2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_npi_present_2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_only_npi_licensor_present-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_only_npi_licensor_present-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_only_npi_scope-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_only_npi_scope-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_only_npi_scope-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_only_npi_scope-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_passive_1-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_passive_1-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_passive_1-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_passive_1-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_passive_2-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_passive_2-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_passive_2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_passive_2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_c_command-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_c_command-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_c_command-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_c_command-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_case_1-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_case_1-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_case_1-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_case_1-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_case_2-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_case_2-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_case_2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_case_2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_domain_1-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_domain_1-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_domain_1-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_domain_1-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_domain_2-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_domain_2-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_domain_2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_domain_2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_domain_3-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_domain_3-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_domain_3-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_domain_3-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_principle_A_reconstruction-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_principle_A_reconstruction-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_sentential_subject_island-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_sentential_subject_island-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_superlative_quantifiers_1-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_superlative_quantifiers_1-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_superlative_quantifiers_2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_superlative_quantifiers_2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_tough_vs_raising_1-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_tough_vs_raising_1-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_tough_vs_raising_1-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_tough_vs_raising_1-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_tough_vs_raising_2-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_tough_vs_raising_2-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_tough_vs_raising_2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_tough_vs_raising_2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_transitive-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_transitive-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_transitive-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_transitive-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_wh_island-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_wh_island-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_wh_island-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_wh_island-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_wh_questions_object_gap-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_wh_questions_object_gap-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_wh_questions_object_gap-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_wh_questions_object_gap-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_wh_questions_subject_gap-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_wh_questions_subject_gap-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_wh_vs_that_no_gap-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_wh_vs_that_no_gap-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_wh_vs_that_no_gap-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_wh_vs_that_no_gap-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/blimp_wh_vs_that_with_gap-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_wh_vs_that_with_gap-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/blimp_wh_vs_that_with_gap-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/blimp_wh_vs_that_with_gap-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/boolq-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/boolq-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/boolq-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/boolq-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/boolq-v1-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/boolq-v1-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/boolq-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/boolq-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/cb-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/cb-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/cb-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/cb-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/cb-v1-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/cb-v1-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/cb-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/cb-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/cola-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/cola-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/cola-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/cola-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/copa-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/copa-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/copa-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/copa-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/coqa-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/coqa-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/coqa-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/coqa-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/coqa-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/coqa-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/coqa-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/coqa-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_age-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_age-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_age-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_age-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_autre-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_autre-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_autre-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_autre-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_disability-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_disability-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_gender-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_gender-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_gender-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_gender-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_nationality-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_nationality-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_race_color-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_race_color-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_religion-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_religion-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_religion-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_religion-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_english_socioeconomic-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_english_socioeconomic-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_age-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_age-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_age-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_age-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_autre-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_autre-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_autre-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_autre-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_disability-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_disability-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_gender-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_gender-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_gender-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_gender-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_nationality-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_nationality-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_race_color-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_race_color-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_religion-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_religion-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_religion-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_religion-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/crows_pairs_french_socioeconomic-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/crows_pairs_french_socioeconomic-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/cycle_letters-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/cycle_letters-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/cycle_letters-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/cycle_letters-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/drop-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/drop-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/drop-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/drop-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/drop-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/drop-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/drop-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/drop-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/ethics_cm-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/ethics_cm-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/ethics_cm-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/ethics_cm-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/ethics_deontology-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/ethics_deontology-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/ethics_deontology-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/ethics_deontology-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/ethics_justice-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/ethics_justice-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/ethics_justice-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/ethics_justice-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/ethics_utilitarianism-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/ethics_utilitarianism-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/ethics_utilitarianism-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/ethics_utilitarianism-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/ethics_utilitarianism_original-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/ethics_utilitarianism_original-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/ethics_virtue-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/ethics_virtue-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/ethics_virtue-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/ethics_virtue-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/gsm8k-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/gsm8k-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/gsm8k-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/gsm8k-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/headqa-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/headqa-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/headqa-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/headqa-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/headqa_en-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/headqa_en-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/headqa_en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/headqa_en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/headqa_es-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/headqa_es-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/headqa_es-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/headqa_es-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hellaswag-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hellaswag-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hellaswag-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hellaswag-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-abstract_algebra-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-abstract_algebra-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-anatomy-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-anatomy-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-anatomy-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-anatomy-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-astronomy-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-astronomy-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-astronomy-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-astronomy-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-business_ethics-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-business_ethics-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-clinical_knowledge-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-clinical_knowledge-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-college_biology-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-college_biology-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-college_chemistry-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-college_chemistry-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-college_medicine-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-college_medicine-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-college_physics-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-college_physics-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-computer_security-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-computer_security-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-conceptual_physics-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-conceptual_physics-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-econometrics-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-econometrics-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-econometrics-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-econometrics-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-formal_logic-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-formal_logic-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-formal_logic-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-formal_logic-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-global_facts-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-global_facts-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-global_facts-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-global_facts-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-human_aging-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-human_aging-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-human_aging-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-human_aging-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-human_sexuality-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-human_sexuality-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-international_law-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-international_law-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-jurisprudence-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-jurisprudence-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-jurisprudence-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-jurisprudence-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-logical_fallacies-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-logical_fallacies-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-machine_learning-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-machine_learning-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-management-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-management-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-management-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-management-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-marketing-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-marketing-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-marketing-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-marketing-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-medical_genetics-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-medical_genetics-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-miscellaneous-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-miscellaneous-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-miscellaneous-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-miscellaneous-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-moral_disputes-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-moral_disputes-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-moral_scenarios-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-moral_scenarios-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-nutrition-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-nutrition-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-nutrition-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-nutrition-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-philosophy-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-philosophy-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-philosophy-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-philosophy-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-prehistory-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-prehistory-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-prehistory-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-prehistory-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-professional_law-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-professional_law-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-public_relations-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-public_relations-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-security_studies-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-security_studies-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-sociology-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-sociology-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-sociology-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-sociology-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-us_foreign_policy-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-us_foreign_policy-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-virology-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-virology-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-virology-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-virology-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/hendrycksTest-world_religions-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/hendrycksTest-world_religions-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/iwslt17-ar-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/iwslt17-ar-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/iwslt17-ar-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/iwslt17-ar-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/iwslt17-en-ar-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/iwslt17-en-ar-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/iwslt17-en-ar-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/iwslt17-en-ar-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_cloze-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_cloze-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_cloze-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_cloze-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_mt_de-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_mt_de-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_mt_de-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_mt_de-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_mt_en-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_mt_en-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_mt_en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_mt_en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_mt_es-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_mt_es-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_mt_es-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_mt_es-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_mt_fr-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_mt_fr-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_mt_fr-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_mt_fr-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_mt_it-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_mt_it-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_mt_it-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_mt_it-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_openai-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_openai-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_cloze-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_cloze-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_cloze-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_cloze-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_mt_de-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_mt_de-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_mt_de-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_mt_de-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_mt_en-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_mt_en-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_mt_en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_mt_en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_mt_es-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_mt_es-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_mt_es-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_mt_es-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_mt_fr-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_mt_fr-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_mt_fr-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_mt_fr-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_mt_it-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_mt_it-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_openai_mt_it-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_openai_mt_it-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_standard-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_standard-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_standard-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_standard-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/lambada_standard_cloze-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_standard_cloze-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/lambada_standard_cloze-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/lambada_standard_cloze-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/logiqa-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/logiqa-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/logiqa-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/logiqa-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/math_algebra-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_algebra-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_algebra-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_algebra-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/math_algebra-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_algebra-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_algebra-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_algebra-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/math_counting_and_prob-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_counting_and_prob-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_counting_and_prob-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_counting_and_prob-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/math_counting_and_prob-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_counting_and_prob-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_counting_and_prob-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_counting_and_prob-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/math_geometry-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_geometry-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_geometry-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_geometry-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/math_geometry-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_geometry-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_geometry-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_geometry-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/math_intermediate_algebra-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_intermediate_algebra-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_intermediate_algebra-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_intermediate_algebra-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/math_intermediate_algebra-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_intermediate_algebra-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_intermediate_algebra-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_intermediate_algebra-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/math_num_theory-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_num_theory-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_num_theory-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_num_theory-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/math_num_theory-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_num_theory-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_num_theory-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_num_theory-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/math_prealgebra-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_prealgebra-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_prealgebra-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_prealgebra-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/math_prealgebra-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_prealgebra-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_prealgebra-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_prealgebra-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/math_precalc-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_precalc-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_precalc-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_precalc-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/math_precalc-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_precalc-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/math_precalc-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/math_precalc-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/mathqa-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mathqa-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/mathqa-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mathqa-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/mc_taco-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mc_taco-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/mc_taco-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mc_taco-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/mnli-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mnli-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/mnli-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mnli-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/mnli_mismatched-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mnli_mismatched-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/mnli_mismatched-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mnli_mismatched-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/mrpc-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mrpc-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/mrpc-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mrpc-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/multirc-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/multirc-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/multirc-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/multirc-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/multirc-v1-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/multirc-v1-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/multirc-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/multirc-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/mutual-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mutual-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/mutual-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mutual-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/mutual-v1-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mutual-v1-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/mutual-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mutual-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/mutual_plus-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mutual_plus-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/mutual_plus-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mutual_plus-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/mutual_plus-v1-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mutual_plus-v1-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/mutual_plus-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/mutual_plus-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/openbookqa-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/openbookqa-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/openbookqa-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/openbookqa-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_arxiv-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_arxiv-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_arxiv-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_arxiv-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_arxiv-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_arxiv-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_arxiv-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_arxiv-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_bookcorpus2-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_bookcorpus2-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_bookcorpus2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_bookcorpus2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_bookcorpus2-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_bookcorpus2-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_bookcorpus2-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_bookcorpus2-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_books3-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_books3-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_books3-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_books3-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_books3-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_books3-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_books3-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_books3-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_dm-mathematics-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_dm-mathematics-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_dm-mathematics-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_dm-mathematics-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_dm-mathematics-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_dm-mathematics-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_dm-mathematics-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_dm-mathematics-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_enron-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_enron-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_enron-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_enron-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_enron-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_enron-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_enron-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_enron-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_europarl-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_europarl-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_europarl-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_europarl-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_europarl-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_europarl-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_europarl-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_europarl-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_freelaw-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_freelaw-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_freelaw-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_freelaw-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_freelaw-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_freelaw-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_freelaw-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_freelaw-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_github-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_github-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_github-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_github-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_github-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_github-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_github-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_github-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_gutenberg-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_gutenberg-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_gutenberg-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_gutenberg-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_gutenberg-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_gutenberg-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_gutenberg-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_gutenberg-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_hackernews-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_hackernews-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_hackernews-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_hackernews-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_hackernews-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_hackernews-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_hackernews-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_hackernews-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_nih-exporter-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_nih-exporter-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_nih-exporter-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_nih-exporter-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_nih-exporter-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_nih-exporter-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_nih-exporter-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_nih-exporter-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_opensubtitles-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_opensubtitles-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_opensubtitles-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_opensubtitles-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_opensubtitles-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_opensubtitles-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_opensubtitles-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_opensubtitles-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_openwebtext2-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_openwebtext2-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_openwebtext2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_openwebtext2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_openwebtext2-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_openwebtext2-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_openwebtext2-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_openwebtext2-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_philpapers-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_philpapers-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_philpapers-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_philpapers-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_philpapers-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_philpapers-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_philpapers-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_philpapers-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_pile-cc-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_pile-cc-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_pile-cc-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_pile-cc-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_pile-cc-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_pile-cc-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_pile-cc-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_pile-cc-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_pubmed-abstracts-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_pubmed-abstracts-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_pubmed-abstracts-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_pubmed-abstracts-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_pubmed-central-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_pubmed-central-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_pubmed-central-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_pubmed-central-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_pubmed-central-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_pubmed-central-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_pubmed-central-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_pubmed-central-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_stackexchange-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_stackexchange-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_stackexchange-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_stackexchange-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_stackexchange-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_stackexchange-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_stackexchange-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_stackexchange-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_ubuntu-irc-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_ubuntu-irc-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_ubuntu-irc-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_ubuntu-irc-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_ubuntu-irc-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_ubuntu-irc-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_ubuntu-irc-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_ubuntu-irc-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_uspto-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_uspto-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_uspto-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_uspto-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_uspto-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_uspto-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_uspto-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_uspto-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_wikipedia-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_wikipedia-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_wikipedia-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_wikipedia-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_wikipedia-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_wikipedia-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/pile_wikipedia-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_wikipedia-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_youtubesubtitles-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_youtubesubtitles-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pile_youtubesubtitles-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pile_youtubesubtitles-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/piqa-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/piqa-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/piqa-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/piqa-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/prost-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/prost-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/prost-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/prost-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/pubmedqa-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pubmedqa-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/pubmedqa-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/pubmedqa-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/qa4mre_2011-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/qa4mre_2011-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/qa4mre_2011-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/qa4mre_2011-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/qa4mre_2012-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/qa4mre_2012-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/qa4mre_2012-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/qa4mre_2012-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/qa4mre_2013-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/qa4mre_2013-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/qa4mre_2013-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/qa4mre_2013-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/qnli-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/qnli-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/qnli-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/qnli-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/qqp-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/qqp-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/qqp-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/qqp-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/race-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/race-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/race-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/race-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/random_insertion-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/random_insertion-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/random_insertion-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/random_insertion-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/record-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/record-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/record-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/record-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/reversed_words-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/reversed_words-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/reversed_words-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/reversed_words-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/rte-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/rte-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/rte-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/rte-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/sciq-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/sciq-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/sciq-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/sciq-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/squad2-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/squad2-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/squad2-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/squad2-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/squad2-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/squad2-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/squad2-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/squad2-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/squad2-v1-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/squad2-v1-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/squad2-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/squad2-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/sst-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/sst-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/sst-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/sst-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/swag-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/swag-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/swag-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/swag-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/toxigen-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/toxigen-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/toxigen-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/toxigen-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/triviaqa-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/triviaqa-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/triviaqa-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/triviaqa-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/triviaqa-v1-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/triviaqa-v1-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/triviaqa-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/triviaqa-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/truthfulqa_gen-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/truthfulqa_gen-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/truthfulqa_gen-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/truthfulqa_gen-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/truthfulqa_gen-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/truthfulqa_gen-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/truthfulqa_gen-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/truthfulqa_gen-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/truthfulqa_mc-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/truthfulqa_mc-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/truthfulqa_mc-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/truthfulqa_mc-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/truthfulqa_mc-v1-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/truthfulqa_mc-v1-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/truthfulqa_mc-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/truthfulqa_mc-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/webqs-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/webqs-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/webqs-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/webqs-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wic-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wic-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/wic-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wic-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wikitext-v0-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wikitext-v0-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/wikitext-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wikitext-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wikitext-v1-loglikelihood_rolling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wikitext-v1-loglikelihood_rolling -------------------------------------------------------------------------------- /tests/testdata/wikitext-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wikitext-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/winogrande-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/winogrande-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/winogrande-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/winogrande-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt14-en-fr-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt14-en-fr-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt14-en-fr-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt14-en-fr-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt14-fr-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt14-fr-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt14-fr-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt14-fr-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt16-de-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt16-de-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt16-de-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt16-de-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt16-en-de-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt16-en-de-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt16-en-de-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt16-en-de-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt16-en-ro-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt16-en-ro-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt16-en-ro-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt16-en-ro-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt16-ro-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt16-ro-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt16-ro-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt16-ro-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-cs-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-cs-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-cs-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-cs-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-de-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-de-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-de-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-de-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-de-fr-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-de-fr-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-de-fr-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-de-fr-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-cs-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-cs-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-cs-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-cs-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-de-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-de-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-de-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-de-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-iu-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-iu-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-iu-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-iu-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-ja-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-ja-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-ja-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-ja-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-ja-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-ja-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-ja-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-ja-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-km-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-km-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-km-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-km-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-pl-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-pl-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-pl-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-pl-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-ps-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-ps-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-ps-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-ps-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-ru-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-ru-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-ru-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-ru-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-ta-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-ta-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-ta-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-ta-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-zh-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-zh-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-zh-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-zh-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-zh-v1-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-zh-v1-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-en-zh-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-en-zh-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-fr-de-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-fr-de-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-fr-de-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-fr-de-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-iu-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-iu-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-iu-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-iu-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-ja-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-ja-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-ja-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-ja-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-km-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-km-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-km-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-km-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-pl-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-pl-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-pl-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-pl-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-ps-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-ps-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-ps-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-ps-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-ru-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-ru-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-ru-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-ru-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-ta-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-ta-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-ta-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-ta-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wmt20-zh-en-v0-greedy_until: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-zh-en-v0-greedy_until -------------------------------------------------------------------------------- /tests/testdata/wmt20-zh-en-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wmt20-zh-en-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wnli-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wnli-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/wnli-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wnli-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wnli-v1-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wnli-v1-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/wnli-v1-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wnli-v1-res.json -------------------------------------------------------------------------------- /tests/testdata/wsc-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wsc-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/wsc-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wsc-v0-res.json -------------------------------------------------------------------------------- /tests/testdata/wsc273-v0-loglikelihood: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wsc273-v0-loglikelihood -------------------------------------------------------------------------------- /tests/testdata/wsc273-v0-res.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stability-AI/lm-evaluation-harness/HEAD/tests/testdata/wsc273-v0-res.json --------------------------------------------------------------------------------