├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── data ├── retrieval │ └── cache │ │ └── test.json ├── retrobust │ └── example_questions │ │ └── example_nq_questions.csv └── test_datasets │ └── nq │ └── nq-dev.qa.csv ├── demo.ipynb ├── images └── overview2.png ├── indexer.log ├── requirements.txt └── src ├── common ├── abstract_factory.py ├── config.py ├── dataset_utils.py └── logger.py ├── config ├── 2wikihop │ └── config_with_retrieval_contexts_first.json ├── retrobust │ └── nq │ │ ├── no_retrieval.json │ │ ├── with_retrieval_mix.json │ │ ├── with_retrieval_top_1.json │ │ └── with_retrieval_top_10.json └── strategyqa │ └── config_with_retrieval_contexts_first.json ├── consts.py ├── dataclasses.py ├── dataset_readers ├── dataset_readers_factory.py └── readers │ ├── bamboogle_reader.py │ ├── dataset_reader.py │ ├── fermi_reader.py │ ├── feverous_reader.py │ ├── hotpotqa_reader.py │ ├── nq_reader.py │ ├── quartz_reader.py │ ├── strategy_qa.py │ └── wikihop_reader.py ├── experiments ├── demo.py ├── e2e │ ├── indexer.log │ └── run.py └── retrobust │ ├── README.md │ ├── indexer.log │ └── run.py ├── gpt3_accessors ├── gpt3_accessors │ ├── gpt_accessor_base.py │ ├── gpt_accessor_simple.py │ ├── gpt_accessor_simple_retrobust.py │ ├── gpt_accessor_with_retrieval.py │ ├── gpt_accessor_with_retrieval_context_first.py │ └── gpt_accessor_with_retriever_context_first_retrobust.py └── gpt_accessor_factory.py ├── inference └── wrappers │ ├── base_inference_wrapper.py │ ├── fastchat.py │ └── openai.py ├── opeanai └── utils.py ├── pred_evaluators ├── evaluation.py ├── evaluators_factory.py └── pred_evaluators │ ├── bamboogle_evaluator.py │ ├── base_evaluator.py │ ├── em_evaluator.py │ ├── fermi_evaluator.py │ ├── hover_evaluator.py │ ├── nq_em.py │ ├── quartz_em_evaluator.py │ └── wikihop_evaluator.py ├── prompting ├── prompt_factory.py ├── prompts_to_keep.py └── retrobust.py └── serpapi ├── serpapi.py └── wikipedia.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/README.md -------------------------------------------------------------------------------- /data/retrieval/cache/test.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/retrobust/example_questions/example_nq_questions.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/data/retrobust/example_questions/example_nq_questions.csv -------------------------------------------------------------------------------- /data/test_datasets/nq/nq-dev.qa.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/data/test_datasets/nq/nq-dev.qa.csv -------------------------------------------------------------------------------- /demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/demo.ipynb -------------------------------------------------------------------------------- /images/overview2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/images/overview2.png -------------------------------------------------------------------------------- /indexer.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/requirements.txt -------------------------------------------------------------------------------- /src/common/abstract_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/common/abstract_factory.py -------------------------------------------------------------------------------- /src/common/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/common/config.py -------------------------------------------------------------------------------- /src/common/dataset_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/common/dataset_utils.py -------------------------------------------------------------------------------- /src/common/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/common/logger.py -------------------------------------------------------------------------------- /src/config/2wikihop/config_with_retrieval_contexts_first.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/config/2wikihop/config_with_retrieval_contexts_first.json -------------------------------------------------------------------------------- /src/config/retrobust/nq/no_retrieval.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/config/retrobust/nq/no_retrieval.json -------------------------------------------------------------------------------- /src/config/retrobust/nq/with_retrieval_mix.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/config/retrobust/nq/with_retrieval_mix.json -------------------------------------------------------------------------------- /src/config/retrobust/nq/with_retrieval_top_1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/config/retrobust/nq/with_retrieval_top_1.json -------------------------------------------------------------------------------- /src/config/retrobust/nq/with_retrieval_top_10.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/config/retrobust/nq/with_retrieval_top_10.json -------------------------------------------------------------------------------- /src/config/strategyqa/config_with_retrieval_contexts_first.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/config/strategyqa/config_with_retrieval_contexts_first.json -------------------------------------------------------------------------------- /src/consts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/consts.py -------------------------------------------------------------------------------- /src/dataclasses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/dataclasses.py -------------------------------------------------------------------------------- /src/dataset_readers/dataset_readers_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/dataset_readers/dataset_readers_factory.py -------------------------------------------------------------------------------- /src/dataset_readers/readers/bamboogle_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/dataset_readers/readers/bamboogle_reader.py -------------------------------------------------------------------------------- /src/dataset_readers/readers/dataset_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/dataset_readers/readers/dataset_reader.py -------------------------------------------------------------------------------- /src/dataset_readers/readers/fermi_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/dataset_readers/readers/fermi_reader.py -------------------------------------------------------------------------------- /src/dataset_readers/readers/feverous_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/dataset_readers/readers/feverous_reader.py -------------------------------------------------------------------------------- /src/dataset_readers/readers/hotpotqa_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/dataset_readers/readers/hotpotqa_reader.py -------------------------------------------------------------------------------- /src/dataset_readers/readers/nq_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/dataset_readers/readers/nq_reader.py -------------------------------------------------------------------------------- /src/dataset_readers/readers/quartz_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/dataset_readers/readers/quartz_reader.py -------------------------------------------------------------------------------- /src/dataset_readers/readers/strategy_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/dataset_readers/readers/strategy_qa.py -------------------------------------------------------------------------------- /src/dataset_readers/readers/wikihop_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/dataset_readers/readers/wikihop_reader.py -------------------------------------------------------------------------------- /src/experiments/demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/experiments/demo.py -------------------------------------------------------------------------------- /src/experiments/e2e/indexer.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/experiments/e2e/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/experiments/e2e/run.py -------------------------------------------------------------------------------- /src/experiments/retrobust/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/experiments/retrobust/README.md -------------------------------------------------------------------------------- /src/experiments/retrobust/indexer.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/experiments/retrobust/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/experiments/retrobust/run.py -------------------------------------------------------------------------------- /src/gpt3_accessors/gpt3_accessors/gpt_accessor_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/gpt3_accessors/gpt3_accessors/gpt_accessor_base.py -------------------------------------------------------------------------------- /src/gpt3_accessors/gpt3_accessors/gpt_accessor_simple.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/gpt3_accessors/gpt3_accessors/gpt_accessor_simple.py -------------------------------------------------------------------------------- /src/gpt3_accessors/gpt3_accessors/gpt_accessor_simple_retrobust.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/gpt3_accessors/gpt3_accessors/gpt_accessor_simple_retrobust.py -------------------------------------------------------------------------------- /src/gpt3_accessors/gpt3_accessors/gpt_accessor_with_retrieval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/gpt3_accessors/gpt3_accessors/gpt_accessor_with_retrieval.py -------------------------------------------------------------------------------- /src/gpt3_accessors/gpt3_accessors/gpt_accessor_with_retrieval_context_first.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/gpt3_accessors/gpt3_accessors/gpt_accessor_with_retrieval_context_first.py -------------------------------------------------------------------------------- /src/gpt3_accessors/gpt3_accessors/gpt_accessor_with_retriever_context_first_retrobust.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/gpt3_accessors/gpt3_accessors/gpt_accessor_with_retriever_context_first_retrobust.py -------------------------------------------------------------------------------- /src/gpt3_accessors/gpt_accessor_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/gpt3_accessors/gpt_accessor_factory.py -------------------------------------------------------------------------------- /src/inference/wrappers/base_inference_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/inference/wrappers/base_inference_wrapper.py -------------------------------------------------------------------------------- /src/inference/wrappers/fastchat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/inference/wrappers/fastchat.py -------------------------------------------------------------------------------- /src/inference/wrappers/openai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/inference/wrappers/openai.py -------------------------------------------------------------------------------- /src/opeanai/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/opeanai/utils.py -------------------------------------------------------------------------------- /src/pred_evaluators/evaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/pred_evaluators/evaluation.py -------------------------------------------------------------------------------- /src/pred_evaluators/evaluators_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/pred_evaluators/evaluators_factory.py -------------------------------------------------------------------------------- /src/pred_evaluators/pred_evaluators/bamboogle_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/pred_evaluators/pred_evaluators/bamboogle_evaluator.py -------------------------------------------------------------------------------- /src/pred_evaluators/pred_evaluators/base_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/pred_evaluators/pred_evaluators/base_evaluator.py -------------------------------------------------------------------------------- /src/pred_evaluators/pred_evaluators/em_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/pred_evaluators/pred_evaluators/em_evaluator.py -------------------------------------------------------------------------------- /src/pred_evaluators/pred_evaluators/fermi_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/pred_evaluators/pred_evaluators/fermi_evaluator.py -------------------------------------------------------------------------------- /src/pred_evaluators/pred_evaluators/hover_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/pred_evaluators/pred_evaluators/hover_evaluator.py -------------------------------------------------------------------------------- /src/pred_evaluators/pred_evaluators/nq_em.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/pred_evaluators/pred_evaluators/nq_em.py -------------------------------------------------------------------------------- /src/pred_evaluators/pred_evaluators/quartz_em_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/pred_evaluators/pred_evaluators/quartz_em_evaluator.py -------------------------------------------------------------------------------- /src/pred_evaluators/pred_evaluators/wikihop_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/pred_evaluators/pred_evaluators/wikihop_evaluator.py -------------------------------------------------------------------------------- /src/prompting/prompt_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/prompting/prompt_factory.py -------------------------------------------------------------------------------- /src/prompting/prompts_to_keep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/prompting/prompts_to_keep.py -------------------------------------------------------------------------------- /src/prompting/retrobust.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/prompting/retrobust.py -------------------------------------------------------------------------------- /src/serpapi/serpapi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/serpapi/serpapi.py -------------------------------------------------------------------------------- /src/serpapi/wikipedia.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oriyor/reasoning-on-cots/HEAD/src/serpapi/wikipedia.py --------------------------------------------------------------------------------