├── .gitignore ├── LICENSE.md ├── README.md ├── automatic_prompt_engineer ├── __init__.py ├── ape.py ├── config.py ├── configs │ ├── bandits.yaml │ └── default.yaml ├── data.py ├── evaluate.py ├── evaluation │ ├── __init__.py │ ├── bandits.py │ └── likelihood.py ├── generate.py ├── llm.py └── template.py ├── data └── bigbench-ii │ ├── causal_judgment │ └── task.json │ ├── disambiguation_qa │ └── task.json │ ├── dyck_languages │ └── task.json │ ├── epistemic_reasoning │ └── task.json │ ├── gender_inclusive_sentences_german │ └── task.json │ ├── hyperbaton │ └── task.json │ ├── implicatures │ └── task.json │ ├── linguistics_puzzles │ └── task.json │ ├── logical_fallacy_detection │ └── task.json │ ├── movie_recommendation │ └── task.json │ ├── navigate │ └── task.json │ ├── object_counting │ └── task.json │ ├── operators │ └── task.json │ ├── presuppositions_as_nli │ └── task.json │ ├── question_selection │ └── task.json │ ├── ruin_names │ └── task.json │ ├── snarks │ └── task.json │ ├── sports_understanding │ ├── generate_tasks.py │ ├── task.json │ └── vocab.json │ ├── tense │ └── task.json │ ├── timedial │ └── task.json │ ├── winowhy │ └── task.json │ ├── word_sorting │ └── task.json │ └── word_unscrambling │ └── task.json ├── demo.ipynb ├── demo.py ├── experiments ├── __init__.py ├── configs │ ├── instruction_induction.yaml │ └── truthful_qa.yaml ├── data │ ├── TruthfulQA │ │ ├── TruthfulQA.csv │ │ ├── TruthfulQA_test.csv │ │ ├── TruthfulQA_train.csv │ │ └── load_data.py │ ├── __init__.py │ └── instruction_induction │ │ ├── README.md │ │ ├── annotations │ │ ├── active_to_passive.json │ │ ├── antonyms.json │ │ ├── cause_and_effect.json │ │ ├── common_concept.json │ │ ├── diff.json │ │ ├── first_word_letter.json │ │ ├── informal_to_formal.json │ │ ├── larger_animal.json │ │ ├── letters_list.json │ │ ├── negation.json │ │ ├── num_to_verbal.json │ │ ├── orthography_starts_with.json │ │ ├── rhymes.json │ │ ├── second_word_letter.json │ │ ├── sentence_similarity.json │ │ ├── sentiment.json │ │ ├── singular_to_plural.json │ │ ├── sum.json │ │ ├── synonyms.json │ │ ├── taxonomy_animal.json │ │ ├── translation_en-de.json │ │ ├── translation_en-es.json │ │ ├── translation_en-fr.json │ │ └── word_in_context.json │ │ ├── induction_input │ │ ├── active_to_passive.json │ │ ├── antonyms.json │ │ ├── cause_and_effect.json │ │ ├── common_concept.json │ │ ├── diff.json │ │ ├── first_word_letter.json │ │ ├── informal_to_formal.json │ │ ├── larger_animal.json │ │ ├── letters_list.json │ │ ├── negation.json │ │ ├── num_to_verbal.json │ │ ├── orthography_starts_with.json │ │ ├── rhymes.json │ │ ├── second_word_letter.json │ │ ├── sentence_similarity.json │ │ ├── sentiment.json │ │ ├── singular_to_plural.json │ │ ├── sum.json │ │ ├── synonyms.json │ │ ├── taxonomy_animal.json │ │ ├── translation_en-de.json │ │ ├── translation_en-es.json │ │ ├── translation_en-fr.json │ │ └── word_in_context.json │ │ ├── load_data.py │ │ └── raw │ │ ├── execute │ │ ├── active_to_passive.json │ │ ├── antonyms.json │ │ ├── cause_and_effect.json │ │ ├── common_concept.json │ │ ├── diff.json │ │ ├── first_word_letter.json │ │ ├── informal_to_formal.json │ │ ├── larger_animal.json │ │ ├── letters_list.json │ │ ├── negation.json │ │ ├── num_to_verbal.json │ │ ├── orthography_starts_with.json │ │ ├── rhymes.json │ │ ├── second_word_letter.json │ │ ├── sentence_similarity.json │ │ ├── sentiment.json │ │ ├── singular_to_plural.json │ │ ├── sum.json │ │ ├── synonyms.json │ │ ├── taxonomy_animal.json │ │ ├── translation_en-de.json │ │ ├── translation_en-es.json │ │ ├── translation_en-fr.json │ │ └── word_in_context.json │ │ └── induce │ │ ├── active_to_passive.json │ │ ├── antonyms.json │ │ ├── cause_and_effect.json │ │ ├── common_concept.json │ │ ├── diff.json │ │ ├── first_word_letter.json │ │ ├── informal_to_formal.json │ │ ├── larger_animal.json │ │ ├── letters_list.json │ │ ├── negation.json │ │ ├── num_to_verbal.json │ │ ├── orthography_starts_with.json │ │ ├── rhymes.json │ │ ├── second_word_letter.json │ │ ├── sentence_similarity.json │ │ ├── sentiment.json │ │ ├── singular_to_plural.json │ │ ├── sum.json │ │ ├── synonyms.json │ │ ├── taxonomy_animal.json │ │ ├── translation_en-de.json │ │ ├── translation_en-es.json │ │ ├── translation_en-fr.json │ │ └── word_in_context.json ├── evaluation │ ├── TruthfulQA │ │ └── fine_tuned_gpt_eval.py │ ├── __init__.py │ └── instruction_induction │ │ ├── __init__.py │ │ ├── exec_accuracy.py │ │ └── utility.py ├── run_instruction_induction.py └── run_truthful_qa.py ├── setup.py └── tests ├── __init__.py ├── test_bandits.py ├── test_config.py ├── test_evaluate.py ├── test_generate.py ├── test_llm.py └── test_template.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/LICENSE.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/README.md -------------------------------------------------------------------------------- /automatic_prompt_engineer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /automatic_prompt_engineer/ape.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/automatic_prompt_engineer/ape.py -------------------------------------------------------------------------------- /automatic_prompt_engineer/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/automatic_prompt_engineer/config.py -------------------------------------------------------------------------------- /automatic_prompt_engineer/configs/bandits.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/automatic_prompt_engineer/configs/bandits.yaml -------------------------------------------------------------------------------- /automatic_prompt_engineer/configs/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/automatic_prompt_engineer/configs/default.yaml -------------------------------------------------------------------------------- /automatic_prompt_engineer/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/automatic_prompt_engineer/data.py -------------------------------------------------------------------------------- /automatic_prompt_engineer/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/automatic_prompt_engineer/evaluate.py -------------------------------------------------------------------------------- /automatic_prompt_engineer/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /automatic_prompt_engineer/evaluation/bandits.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/automatic_prompt_engineer/evaluation/bandits.py -------------------------------------------------------------------------------- /automatic_prompt_engineer/evaluation/likelihood.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/automatic_prompt_engineer/evaluation/likelihood.py -------------------------------------------------------------------------------- /automatic_prompt_engineer/generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/automatic_prompt_engineer/generate.py -------------------------------------------------------------------------------- /automatic_prompt_engineer/llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/automatic_prompt_engineer/llm.py -------------------------------------------------------------------------------- /automatic_prompt_engineer/template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/automatic_prompt_engineer/template.py -------------------------------------------------------------------------------- /data/bigbench-ii/causal_judgment/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/causal_judgment/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/disambiguation_qa/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/disambiguation_qa/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/dyck_languages/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/dyck_languages/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/epistemic_reasoning/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/epistemic_reasoning/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/gender_inclusive_sentences_german/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/gender_inclusive_sentences_german/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/hyperbaton/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/hyperbaton/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/implicatures/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/implicatures/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/linguistics_puzzles/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/linguistics_puzzles/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/logical_fallacy_detection/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/logical_fallacy_detection/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/movie_recommendation/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/movie_recommendation/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/navigate/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/navigate/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/object_counting/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/object_counting/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/operators/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/operators/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/presuppositions_as_nli/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/presuppositions_as_nli/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/question_selection/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/question_selection/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/ruin_names/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/ruin_names/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/snarks/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/snarks/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/sports_understanding/generate_tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/sports_understanding/generate_tasks.py -------------------------------------------------------------------------------- /data/bigbench-ii/sports_understanding/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/sports_understanding/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/sports_understanding/vocab.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/sports_understanding/vocab.json -------------------------------------------------------------------------------- /data/bigbench-ii/tense/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/tense/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/timedial/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/timedial/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/winowhy/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/winowhy/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/word_sorting/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/word_sorting/task.json -------------------------------------------------------------------------------- /data/bigbench-ii/word_unscrambling/task.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/data/bigbench-ii/word_unscrambling/task.json -------------------------------------------------------------------------------- /demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/demo.ipynb -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/demo.py -------------------------------------------------------------------------------- /experiments/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/configs/instruction_induction.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/configs/instruction_induction.yaml -------------------------------------------------------------------------------- /experiments/configs/truthful_qa.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/configs/truthful_qa.yaml -------------------------------------------------------------------------------- /experiments/data/TruthfulQA/TruthfulQA.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/TruthfulQA/TruthfulQA.csv -------------------------------------------------------------------------------- /experiments/data/TruthfulQA/TruthfulQA_test.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/TruthfulQA/TruthfulQA_test.csv -------------------------------------------------------------------------------- /experiments/data/TruthfulQA/TruthfulQA_train.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/TruthfulQA/TruthfulQA_train.csv -------------------------------------------------------------------------------- /experiments/data/TruthfulQA/load_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/TruthfulQA/load_data.py -------------------------------------------------------------------------------- /experiments/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/data/instruction_induction/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/README.md -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/active_to_passive.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/active_to_passive.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/antonyms.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/antonyms.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/cause_and_effect.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/cause_and_effect.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/common_concept.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/common_concept.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/diff.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/diff.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/first_word_letter.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/first_word_letter.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/informal_to_formal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/informal_to_formal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/larger_animal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/larger_animal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/letters_list.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/letters_list.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/negation.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/negation.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/num_to_verbal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/num_to_verbal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/orthography_starts_with.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/orthography_starts_with.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/rhymes.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/rhymes.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/second_word_letter.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/second_word_letter.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/sentence_similarity.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/sentence_similarity.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/sentiment.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/sentiment.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/singular_to_plural.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/singular_to_plural.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/sum.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/sum.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/synonyms.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/synonyms.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/taxonomy_animal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/taxonomy_animal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/translation_en-de.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/translation_en-de.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/translation_en-es.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/translation_en-es.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/translation_en-fr.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/translation_en-fr.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/annotations/word_in_context.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/annotations/word_in_context.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/active_to_passive.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/active_to_passive.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/antonyms.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/antonyms.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/cause_and_effect.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/cause_and_effect.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/common_concept.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/common_concept.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/diff.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/diff.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/first_word_letter.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/first_word_letter.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/informal_to_formal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/informal_to_formal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/larger_animal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/larger_animal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/letters_list.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/letters_list.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/negation.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/negation.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/num_to_verbal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/num_to_verbal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/orthography_starts_with.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/orthography_starts_with.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/rhymes.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/rhymes.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/second_word_letter.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/second_word_letter.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/sentence_similarity.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/sentence_similarity.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/sentiment.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/sentiment.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/singular_to_plural.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/singular_to_plural.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/sum.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/sum.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/synonyms.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/synonyms.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/taxonomy_animal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/taxonomy_animal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/translation_en-de.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/translation_en-de.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/translation_en-es.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/translation_en-es.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/translation_en-fr.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/translation_en-fr.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/induction_input/word_in_context.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/induction_input/word_in_context.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/load_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/load_data.py -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/active_to_passive.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/active_to_passive.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/antonyms.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/antonyms.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/cause_and_effect.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/cause_and_effect.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/common_concept.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/common_concept.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/diff.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/diff.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/first_word_letter.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/first_word_letter.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/informal_to_formal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/informal_to_formal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/larger_animal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/larger_animal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/letters_list.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/letters_list.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/negation.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/negation.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/num_to_verbal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/num_to_verbal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/orthography_starts_with.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/orthography_starts_with.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/rhymes.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/rhymes.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/second_word_letter.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/second_word_letter.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/sentence_similarity.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/sentence_similarity.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/sentiment.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/sentiment.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/singular_to_plural.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/singular_to_plural.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/sum.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/sum.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/synonyms.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/synonyms.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/taxonomy_animal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/taxonomy_animal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/translation_en-de.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/translation_en-de.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/translation_en-es.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/translation_en-es.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/translation_en-fr.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/translation_en-fr.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/execute/word_in_context.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/execute/word_in_context.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/active_to_passive.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/active_to_passive.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/antonyms.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/antonyms.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/cause_and_effect.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/cause_and_effect.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/common_concept.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/common_concept.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/diff.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/diff.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/first_word_letter.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/first_word_letter.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/informal_to_formal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/informal_to_formal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/larger_animal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/larger_animal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/letters_list.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/letters_list.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/negation.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/negation.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/num_to_verbal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/num_to_verbal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/orthography_starts_with.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/orthography_starts_with.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/rhymes.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/rhymes.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/second_word_letter.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/second_word_letter.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/sentence_similarity.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/sentence_similarity.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/sentiment.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/sentiment.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/singular_to_plural.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/singular_to_plural.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/sum.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/sum.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/synonyms.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/synonyms.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/taxonomy_animal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/taxonomy_animal.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/translation_en-de.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/translation_en-de.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/translation_en-es.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/translation_en-es.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/translation_en-fr.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/translation_en-fr.json -------------------------------------------------------------------------------- /experiments/data/instruction_induction/raw/induce/word_in_context.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/data/instruction_induction/raw/induce/word_in_context.json -------------------------------------------------------------------------------- /experiments/evaluation/TruthfulQA/fine_tuned_gpt_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/evaluation/TruthfulQA/fine_tuned_gpt_eval.py -------------------------------------------------------------------------------- /experiments/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/evaluation/instruction_induction/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/evaluation/instruction_induction/exec_accuracy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/evaluation/instruction_induction/exec_accuracy.py -------------------------------------------------------------------------------- /experiments/evaluation/instruction_induction/utility.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/evaluation/instruction_induction/utility.py -------------------------------------------------------------------------------- /experiments/run_instruction_induction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/run_instruction_induction.py -------------------------------------------------------------------------------- /experiments/run_truthful_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/experiments/run_truthful_qa.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/setup.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_bandits.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/tests/test_bandits.py -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/tests/test_config.py -------------------------------------------------------------------------------- /tests/test_evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/tests/test_evaluate.py -------------------------------------------------------------------------------- /tests/test_generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/tests/test_generate.py -------------------------------------------------------------------------------- /tests/test_llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/tests/test_llm.py -------------------------------------------------------------------------------- /tests/test_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keirp/automatic_prompt_engineer/HEAD/tests/test_template.py --------------------------------------------------------------------------------