├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── SUPPORT.md ├── requirements.txt ├── src ├── baseline │ ├── direct │ │ └── direct_prompt.py │ └── retrieve │ │ ├── ceiling.py │ │ ├── decompose.py │ │ ├── direct.py │ │ ├── iter-retgen.py │ │ ├── selfask.py │ │ └── selfrag.py ├── conf │ ├── __init__.py │ └── config.py ├── data_module │ ├── __init__.py │ ├── dataset.py │ └── format.py ├── data_synthesize │ ├── chunk_sampling.py │ ├── gpt_query_chunk_synethesize.py │ ├── graph_driven.py │ ├── negative_sampling.py │ ├── negative_sampling_labeled.py │ ├── negative_token_extraction.py │ ├── next_hop_query_construction.py │ ├── next_hop_query_filtering.py │ ├── prompts │ │ ├── __init__.py │ │ ├── hotpotQA.py │ │ ├── musique.py │ │ ├── negative_token_labeling.py │ │ ├── query_labeling.py │ │ ├── span_labeling.py │ │ ├── token_labeling.py │ │ └── wikimqa.py │ ├── query_decompose.py │ ├── query_driven.py │ ├── span_labeling.py │ ├── token_extraction.py │ ├── token_labeling.py │ └── training_data_synthesize.py ├── efficient_rag │ ├── data │ │ ├── __init__.py │ │ ├── filter_dataset.py │ │ ├── label_only_dataset.py │ │ └── labeler_dataset.py │ ├── filter_training.py │ ├── labeler_training.py │ ├── model │ │ ├── __init__.py │ │ └── model.py │ └── token_weight_avg.py ├── efficientrag_qa.py ├── efficientrag_retrieve.py ├── efficientrag_retrieve.sh ├── evaluation │ ├── correctness.py │ └── retrieve.py ├── language_models │ ├── __init__.py │ ├── aoai.py │ ├── base.py │ ├── cloudgpt │ │ ├── __init__.py │ │ └── cloudgpt_aoai.py │ ├── deepseek.py │ └── llama.py ├── retrievers │ ├── __init__.py │ ├── embeddings │ │ ├── __init__.py │ │ ├── ada_embedding.py │ │ ├── base.py │ │ ├── contriever.py │ │ ├── dense_embedding.py │ │ ├── e5.py │ │ ├── embedder.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ └── normalize_text.py │ ├── multihop_data_extrator.py │ ├── passage_embedder.py │ ├── passage_retriever.py │ ├── utils │ │ ├── __init__.py │ │ └── utils.py │ └── vector_index │ │ ├── __init__.py │ │ ├── base.py │ │ └── faiss_index.py └── utils │ ├── __init__.py │ ├── model.py │ └── utils.py └── static ├── bert_labeler.png └── bert_question.png /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/.gitignore -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/README.md -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/SECURITY.md -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/SUPPORT.md -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/requirements.txt -------------------------------------------------------------------------------- /src/baseline/direct/direct_prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/baseline/direct/direct_prompt.py -------------------------------------------------------------------------------- /src/baseline/retrieve/ceiling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/baseline/retrieve/ceiling.py -------------------------------------------------------------------------------- /src/baseline/retrieve/decompose.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/baseline/retrieve/decompose.py -------------------------------------------------------------------------------- /src/baseline/retrieve/direct.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/baseline/retrieve/direct.py -------------------------------------------------------------------------------- /src/baseline/retrieve/iter-retgen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/baseline/retrieve/iter-retgen.py -------------------------------------------------------------------------------- /src/baseline/retrieve/selfask.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/baseline/retrieve/selfask.py -------------------------------------------------------------------------------- /src/baseline/retrieve/selfrag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/baseline/retrieve/selfrag.py -------------------------------------------------------------------------------- /src/conf/__init__.py: -------------------------------------------------------------------------------- 1 | from .config import * 2 | -------------------------------------------------------------------------------- /src/conf/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/conf/config.py -------------------------------------------------------------------------------- /src/data_module/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_module/__init__.py -------------------------------------------------------------------------------- /src/data_module/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_module/dataset.py -------------------------------------------------------------------------------- /src/data_module/format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_module/format.py -------------------------------------------------------------------------------- /src/data_synthesize/chunk_sampling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/chunk_sampling.py -------------------------------------------------------------------------------- /src/data_synthesize/gpt_query_chunk_synethesize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/gpt_query_chunk_synethesize.py -------------------------------------------------------------------------------- /src/data_synthesize/graph_driven.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/graph_driven.py -------------------------------------------------------------------------------- /src/data_synthesize/negative_sampling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/negative_sampling.py -------------------------------------------------------------------------------- /src/data_synthesize/negative_sampling_labeled.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/negative_sampling_labeled.py -------------------------------------------------------------------------------- /src/data_synthesize/negative_token_extraction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/negative_token_extraction.py -------------------------------------------------------------------------------- /src/data_synthesize/next_hop_query_construction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/next_hop_query_construction.py -------------------------------------------------------------------------------- /src/data_synthesize/next_hop_query_filtering.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/next_hop_query_filtering.py -------------------------------------------------------------------------------- /src/data_synthesize/prompts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/prompts/__init__.py -------------------------------------------------------------------------------- /src/data_synthesize/prompts/hotpotQA.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/prompts/hotpotQA.py -------------------------------------------------------------------------------- /src/data_synthesize/prompts/musique.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/prompts/musique.py -------------------------------------------------------------------------------- /src/data_synthesize/prompts/negative_token_labeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/prompts/negative_token_labeling.py -------------------------------------------------------------------------------- /src/data_synthesize/prompts/query_labeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/prompts/query_labeling.py -------------------------------------------------------------------------------- /src/data_synthesize/prompts/span_labeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/prompts/span_labeling.py -------------------------------------------------------------------------------- /src/data_synthesize/prompts/token_labeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/prompts/token_labeling.py -------------------------------------------------------------------------------- /src/data_synthesize/prompts/wikimqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/prompts/wikimqa.py -------------------------------------------------------------------------------- /src/data_synthesize/query_decompose.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/query_decompose.py -------------------------------------------------------------------------------- /src/data_synthesize/query_driven.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/query_driven.py -------------------------------------------------------------------------------- /src/data_synthesize/span_labeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/span_labeling.py -------------------------------------------------------------------------------- /src/data_synthesize/token_extraction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/token_extraction.py -------------------------------------------------------------------------------- /src/data_synthesize/token_labeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/token_labeling.py -------------------------------------------------------------------------------- /src/data_synthesize/training_data_synthesize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/data_synthesize/training_data_synthesize.py -------------------------------------------------------------------------------- /src/efficient_rag/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/efficient_rag/data/__init__.py -------------------------------------------------------------------------------- /src/efficient_rag/data/filter_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/efficient_rag/data/filter_dataset.py -------------------------------------------------------------------------------- /src/efficient_rag/data/label_only_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/efficient_rag/data/label_only_dataset.py -------------------------------------------------------------------------------- /src/efficient_rag/data/labeler_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/efficient_rag/data/labeler_dataset.py -------------------------------------------------------------------------------- /src/efficient_rag/filter_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/efficient_rag/filter_training.py -------------------------------------------------------------------------------- /src/efficient_rag/labeler_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/efficient_rag/labeler_training.py -------------------------------------------------------------------------------- /src/efficient_rag/model/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import DebertaForSequenceTokenClassification 2 | -------------------------------------------------------------------------------- /src/efficient_rag/model/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/efficient_rag/model/model.py -------------------------------------------------------------------------------- /src/efficient_rag/token_weight_avg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/efficient_rag/token_weight_avg.py -------------------------------------------------------------------------------- /src/efficientrag_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/efficientrag_qa.py -------------------------------------------------------------------------------- /src/efficientrag_retrieve.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/efficientrag_retrieve.py -------------------------------------------------------------------------------- /src/efficientrag_retrieve.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/efficientrag_retrieve.sh -------------------------------------------------------------------------------- /src/evaluation/correctness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/evaluation/correctness.py -------------------------------------------------------------------------------- /src/evaluation/retrieve.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/evaluation/retrieve.py -------------------------------------------------------------------------------- /src/language_models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/language_models/__init__.py -------------------------------------------------------------------------------- /src/language_models/aoai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/language_models/aoai.py -------------------------------------------------------------------------------- /src/language_models/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/language_models/base.py -------------------------------------------------------------------------------- /src/language_models/cloudgpt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/language_models/cloudgpt/__init__.py -------------------------------------------------------------------------------- /src/language_models/cloudgpt/cloudgpt_aoai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/language_models/cloudgpt/cloudgpt_aoai.py -------------------------------------------------------------------------------- /src/language_models/deepseek.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/language_models/deepseek.py -------------------------------------------------------------------------------- /src/language_models/llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/language_models/llama.py -------------------------------------------------------------------------------- /src/retrievers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/__init__.py -------------------------------------------------------------------------------- /src/retrievers/embeddings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/embeddings/__init__.py -------------------------------------------------------------------------------- /src/retrievers/embeddings/ada_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/embeddings/ada_embedding.py -------------------------------------------------------------------------------- /src/retrievers/embeddings/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/embeddings/base.py -------------------------------------------------------------------------------- /src/retrievers/embeddings/contriever.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/embeddings/contriever.py -------------------------------------------------------------------------------- /src/retrievers/embeddings/dense_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/embeddings/dense_embedding.py -------------------------------------------------------------------------------- /src/retrievers/embeddings/e5.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/embeddings/e5.py -------------------------------------------------------------------------------- /src/retrievers/embeddings/embedder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/embeddings/embedder.py -------------------------------------------------------------------------------- /src/retrievers/embeddings/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/retrievers/embeddings/utils/normalize_text.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/embeddings/utils/normalize_text.py -------------------------------------------------------------------------------- /src/retrievers/multihop_data_extrator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/multihop_data_extrator.py -------------------------------------------------------------------------------- /src/retrievers/passage_embedder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/passage_embedder.py -------------------------------------------------------------------------------- /src/retrievers/passage_retriever.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/passage_retriever.py -------------------------------------------------------------------------------- /src/retrievers/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/retrievers/utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/utils/utils.py -------------------------------------------------------------------------------- /src/retrievers/vector_index/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/vector_index/__init__.py -------------------------------------------------------------------------------- /src/retrievers/vector_index/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/vector_index/base.py -------------------------------------------------------------------------------- /src/retrievers/vector_index/faiss_index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/retrievers/vector_index/faiss_index.py -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/utils/__init__.py -------------------------------------------------------------------------------- /src/utils/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/utils/model.py -------------------------------------------------------------------------------- /src/utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/src/utils/utils.py -------------------------------------------------------------------------------- /static/bert_labeler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/static/bert_labeler.png -------------------------------------------------------------------------------- /static/bert_question.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NIL-zhuang/EfficientRAG-official/HEAD/static/bert_question.png --------------------------------------------------------------------------------