├── LICENSE ├── README.md ├── bardcache.py ├── config.py ├── create_data.py ├── dataset.py ├── dune ├── arithmetic.json ├── arithmetic_locality.json ├── debiasing_I.json ├── debiasing_II.json ├── debiasing_II_locality.json ├── debiasing_I_locality.json ├── new_info.json ├── new_info_locality.json ├── scientific.json └── scientific_locality.json ├── dune_figure.jpg ├── eval.py ├── eval_utils.py ├── gptcache.py ├── prompt.py ├── requirements.txt ├── scripts ├── arithmetic │ ├── arithmetic_api_edit_nedit.sh │ ├── arithmetic_api_retrieval.sh │ ├── arithmetic_api_retrieval_conditional.sh │ ├── arithmetic_api_retrieval_gpt3.sh │ ├── arithmetic_api_retrieval_gpt3_num_retrievals.sh │ ├── arithmetic_flan_edit_nedit.sh │ ├── arithmetic_flan_finetuning.sh │ ├── arithmetic_flan_retrieval.sh │ ├── arithmetic_flan_retrieval_conditional.sh │ ├── arithmetic_flan_retrieval_conditional_num_retrievals.sh │ ├── arithmetic_flan_retrieval_gpt3.sh │ ├── arithmetic_flan_retrieval_gpt3_num_retrievals.sh │ ├── arithmetic_flan_retrieval_num_retrievals.sh │ ├── run_arithmetic_flan_templates.sh │ └── run_eval_hf_arithmetic.sh ├── newinfo │ ├── newinfo_api_edit_nedit.sh │ ├── newinfo_api_retrieval.sh │ ├── newinfo_api_retrieval_conditional.sh │ ├── newinfo_api_retrieval_gpt3.sh │ ├── newinfo_flan_edit_nedit.sh │ ├── newinfo_flan_finetuning.sh │ ├── newinfo_flan_retrieval.sh │ ├── newinfo_flan_retrieval_conditional.sh │ ├── newinfo_flan_retrieval_conditional_num_retrieval.sh │ ├── newinfo_flan_retrieval_gpt3.sh │ └── newinfo_flan_retrieval_num_retrievals.sh └── scientific │ ├── arc_api_edit_nedit.sh │ ├── arc_api_retrieval.sh │ ├── arc_api_retrieval_conditional.sh │ ├── arc_api_retrieval_gpt3.sh │ ├── arc_flan_edit_nedit.sh │ ├── arc_flan_finetuning.sh │ ├── arc_flan_retrieval.sh │ ├── arc_flan_retrieval_conditional.sh │ ├── arc_flan_retrieval_gpt3.sh │ ├── arc_llama_edit_nedit.sh │ ├── arc_llama_finetuning.sh │ ├── arc_llama_retrieval.sh │ ├── arc_llama_retrieval_conditional.sh │ └── arc_llama_retrieval_gpt3.sh ├── source ├── arc │ └── chat_prompt_dict.json ├── bbq │ ├── Gender_identity.jsonl │ ├── Race_ethnicity.jsonl │ ├── Religion.jsonl │ └── when_asked_bbq_amb.txt ├── fine-tuning-pool │ ├── all_shuffled_edits.txt │ ├── test_shuffled_edits.txt │ └── train_shuffled_edits.txt └── scope-classifier │ └── all.json └── util.py /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/README.md -------------------------------------------------------------------------------- /bardcache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/bardcache.py -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/config.py -------------------------------------------------------------------------------- /create_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/create_data.py -------------------------------------------------------------------------------- /dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dataset.py -------------------------------------------------------------------------------- /dune/arithmetic.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dune/arithmetic.json -------------------------------------------------------------------------------- /dune/arithmetic_locality.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dune/arithmetic_locality.json -------------------------------------------------------------------------------- /dune/debiasing_I.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dune/debiasing_I.json -------------------------------------------------------------------------------- /dune/debiasing_II.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dune/debiasing_II.json -------------------------------------------------------------------------------- /dune/debiasing_II_locality.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dune/debiasing_II_locality.json -------------------------------------------------------------------------------- /dune/debiasing_I_locality.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dune/debiasing_I_locality.json -------------------------------------------------------------------------------- /dune/new_info.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dune/new_info.json -------------------------------------------------------------------------------- /dune/new_info_locality.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dune/new_info_locality.json -------------------------------------------------------------------------------- /dune/scientific.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dune/scientific.json -------------------------------------------------------------------------------- /dune/scientific_locality.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dune/scientific_locality.json -------------------------------------------------------------------------------- /dune_figure.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/dune_figure.jpg -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/eval.py -------------------------------------------------------------------------------- /eval_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/eval_utils.py -------------------------------------------------------------------------------- /gptcache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/gptcache.py -------------------------------------------------------------------------------- /prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/prompt.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_api_edit_nedit.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_api_edit_nedit.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_api_retrieval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_api_retrieval.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_api_retrieval_conditional.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_api_retrieval_conditional.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_api_retrieval_gpt3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_api_retrieval_gpt3.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_api_retrieval_gpt3_num_retrievals.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_api_retrieval_gpt3_num_retrievals.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_flan_edit_nedit.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_flan_edit_nedit.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_flan_finetuning.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_flan_finetuning.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_flan_retrieval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_flan_retrieval.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_flan_retrieval_conditional.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_flan_retrieval_conditional.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_flan_retrieval_conditional_num_retrievals.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_flan_retrieval_conditional_num_retrievals.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_flan_retrieval_gpt3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_flan_retrieval_gpt3.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_flan_retrieval_gpt3_num_retrievals.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_flan_retrieval_gpt3_num_retrievals.sh -------------------------------------------------------------------------------- /scripts/arithmetic/arithmetic_flan_retrieval_num_retrievals.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/arithmetic_flan_retrieval_num_retrievals.sh -------------------------------------------------------------------------------- /scripts/arithmetic/run_arithmetic_flan_templates.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/run_arithmetic_flan_templates.sh -------------------------------------------------------------------------------- /scripts/arithmetic/run_eval_hf_arithmetic.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/arithmetic/run_eval_hf_arithmetic.sh -------------------------------------------------------------------------------- /scripts/newinfo/newinfo_api_edit_nedit.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/newinfo/newinfo_api_edit_nedit.sh -------------------------------------------------------------------------------- /scripts/newinfo/newinfo_api_retrieval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/newinfo/newinfo_api_retrieval.sh -------------------------------------------------------------------------------- /scripts/newinfo/newinfo_api_retrieval_conditional.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/newinfo/newinfo_api_retrieval_conditional.sh -------------------------------------------------------------------------------- /scripts/newinfo/newinfo_api_retrieval_gpt3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/newinfo/newinfo_api_retrieval_gpt3.sh -------------------------------------------------------------------------------- /scripts/newinfo/newinfo_flan_edit_nedit.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/newinfo/newinfo_flan_edit_nedit.sh -------------------------------------------------------------------------------- /scripts/newinfo/newinfo_flan_finetuning.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/newinfo/newinfo_flan_finetuning.sh -------------------------------------------------------------------------------- /scripts/newinfo/newinfo_flan_retrieval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/newinfo/newinfo_flan_retrieval.sh -------------------------------------------------------------------------------- /scripts/newinfo/newinfo_flan_retrieval_conditional.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/newinfo/newinfo_flan_retrieval_conditional.sh -------------------------------------------------------------------------------- /scripts/newinfo/newinfo_flan_retrieval_conditional_num_retrieval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/newinfo/newinfo_flan_retrieval_conditional_num_retrieval.sh -------------------------------------------------------------------------------- /scripts/newinfo/newinfo_flan_retrieval_gpt3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/newinfo/newinfo_flan_retrieval_gpt3.sh -------------------------------------------------------------------------------- /scripts/newinfo/newinfo_flan_retrieval_num_retrievals.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/newinfo/newinfo_flan_retrieval_num_retrievals.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_api_edit_nedit.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_api_edit_nedit.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_api_retrieval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_api_retrieval.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_api_retrieval_conditional.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_api_retrieval_conditional.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_api_retrieval_gpt3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_api_retrieval_gpt3.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_flan_edit_nedit.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_flan_edit_nedit.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_flan_finetuning.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_flan_finetuning.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_flan_retrieval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_flan_retrieval.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_flan_retrieval_conditional.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_flan_retrieval_conditional.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_flan_retrieval_gpt3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_flan_retrieval_gpt3.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_llama_edit_nedit.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_llama_edit_nedit.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_llama_finetuning.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_llama_finetuning.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_llama_retrieval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_llama_retrieval.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_llama_retrieval_conditional.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_llama_retrieval_conditional.sh -------------------------------------------------------------------------------- /scripts/scientific/arc_llama_retrieval_gpt3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/scripts/scientific/arc_llama_retrieval_gpt3.sh -------------------------------------------------------------------------------- /source/arc/chat_prompt_dict.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/source/arc/chat_prompt_dict.json -------------------------------------------------------------------------------- /source/bbq/Gender_identity.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/source/bbq/Gender_identity.jsonl -------------------------------------------------------------------------------- /source/bbq/Race_ethnicity.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/source/bbq/Race_ethnicity.jsonl -------------------------------------------------------------------------------- /source/bbq/Religion.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/source/bbq/Religion.jsonl -------------------------------------------------------------------------------- /source/bbq/when_asked_bbq_amb.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/source/bbq/when_asked_bbq_amb.txt -------------------------------------------------------------------------------- /source/fine-tuning-pool/all_shuffled_edits.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/source/fine-tuning-pool/all_shuffled_edits.txt -------------------------------------------------------------------------------- /source/fine-tuning-pool/test_shuffled_edits.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/source/fine-tuning-pool/test_shuffled_edits.txt -------------------------------------------------------------------------------- /source/fine-tuning-pool/train_shuffled_edits.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/source/fine-tuning-pool/train_shuffled_edits.txt -------------------------------------------------------------------------------- /source/scope-classifier/all.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/source/scope-classifier/all.json -------------------------------------------------------------------------------- /util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feyzaakyurek/dune/HEAD/util.py --------------------------------------------------------------------------------