├── LICENSE ├── README.md ├── base.py ├── base_1.py ├── base_3.py ├── chunk.ipynb ├── chunk_gpt.py ├── chunk_llamaindex.py ├── chunk_lumberchunker.py ├── chunk_moc.py ├── data └── crud_split │ └── split_merged.json ├── embeddings ├── __pycache__ │ └── base.cpython-310.pyc └── base.py ├── evaldata ├── OmniEval.json └── multifieldqa_zh.json ├── evaluator.py ├── image └── experimental_results.png ├── llms ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── base.cpython-310.pyc │ ├── local_model.cpython-310.pyc │ └── remote_model.cpython-310.pyc └── base.py ├── mom_1.py ├── mom_2.py ├── quick_start.py ├── requirements.txt ├── retrieval.py └── src ├── .cache └── huggingface │ ├── bleu │ ├── README.md │ ├── bleu.py │ ├── nmt_bleu.py │ └── tokenizer_13a.py │ └── rouge │ └── rouge.py ├── configs ├── __pycache__ │ └── config.cpython-310.pyc └── config.py ├── core └── analyst.py ├── datasets ├── __pycache__ │ ├── base.cpython-310.pyc │ └── xinhua.cpython-310.pyc ├── base.py └── xinhua.py ├── embeddings ├── __pycache__ │ └── base.cpython-310.pyc └── base.py ├── eval_retrieval ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ └── base.cpython-310.pyc └── base.py ├── llms ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── base.cpython-310.pyc │ ├── local_model.cpython-310.pyc │ └── remote_model.cpython-310.pyc ├── api_model.py ├── base.py ├── local_model.py └── remote_model.py ├── metric ├── __pycache__ │ ├── common.cpython-310.pyc │ └── quest_eval.cpython-310.pyc ├── common.py └── quest_eval.py ├── prompts ├── continue_writing.txt ├── hallu_mod.txt ├── quest_answer.txt ├── quest_eval_answer.txt ├── quest_eval_gen.txt └── summary.txt ├── quest_eval ├── ContinueWriting_quest_gt_save.json ├── HalluModified_quest_gt_save.json ├── QuestAnswer1Doc_quest_gt_save.json ├── QuestAnswer2Docs_quest_gt_save.json ├── QuestAnswer3Docs_quest_gt_save.json └── Summary_quest_gt_save.json ├── retrievers ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── base.cpython-310.pyc │ ├── bm25.cpython-310.pyc │ ├── hybrid.cpython-310.pyc │ └── hybrid_rerank.cpython-310.pyc ├── base.py ├── base1.py ├── bm25.py ├── hybrid.py └── hybrid_rerank.py └── tasks ├── __pycache__ ├── base.cpython-310.pyc ├── continue_writing.cpython-310.pyc ├── hallucinated_modified.cpython-310.pyc ├── quest_answer.cpython-310.pyc └── summary.cpython-310.pyc ├── base.py ├── continue_writing.py ├── hallucinated_modified.py ├── quest_answer.py └── summary.py /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/README.md -------------------------------------------------------------------------------- /base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/base.py -------------------------------------------------------------------------------- /base_1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/base_1.py -------------------------------------------------------------------------------- /base_3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/base_3.py -------------------------------------------------------------------------------- /chunk.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/chunk.ipynb -------------------------------------------------------------------------------- /chunk_gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/chunk_gpt.py -------------------------------------------------------------------------------- /chunk_llamaindex.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/chunk_llamaindex.py -------------------------------------------------------------------------------- /chunk_lumberchunker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/chunk_lumberchunker.py -------------------------------------------------------------------------------- /chunk_moc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/chunk_moc.py -------------------------------------------------------------------------------- /data/crud_split/split_merged.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/data/crud_split/split_merged.json -------------------------------------------------------------------------------- /embeddings/__pycache__/base.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/embeddings/__pycache__/base.cpython-310.pyc -------------------------------------------------------------------------------- /embeddings/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/embeddings/base.py -------------------------------------------------------------------------------- /evaldata/OmniEval.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/evaldata/OmniEval.json -------------------------------------------------------------------------------- /evaldata/multifieldqa_zh.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/evaldata/multifieldqa_zh.json -------------------------------------------------------------------------------- /evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/evaluator.py -------------------------------------------------------------------------------- /image/experimental_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/image/experimental_results.png -------------------------------------------------------------------------------- /llms/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /llms/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/llms/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /llms/__pycache__/base.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/llms/__pycache__/base.cpython-310.pyc -------------------------------------------------------------------------------- /llms/__pycache__/local_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/llms/__pycache__/local_model.cpython-310.pyc -------------------------------------------------------------------------------- /llms/__pycache__/remote_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/llms/__pycache__/remote_model.cpython-310.pyc -------------------------------------------------------------------------------- /llms/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/llms/base.py -------------------------------------------------------------------------------- /mom_1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/mom_1.py -------------------------------------------------------------------------------- /mom_2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/mom_2.py -------------------------------------------------------------------------------- /quick_start.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/quick_start.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/requirements.txt -------------------------------------------------------------------------------- /retrieval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/retrieval.py -------------------------------------------------------------------------------- /src/.cache/huggingface/bleu/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/.cache/huggingface/bleu/README.md -------------------------------------------------------------------------------- /src/.cache/huggingface/bleu/bleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/.cache/huggingface/bleu/bleu.py -------------------------------------------------------------------------------- /src/.cache/huggingface/bleu/nmt_bleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/.cache/huggingface/bleu/nmt_bleu.py -------------------------------------------------------------------------------- /src/.cache/huggingface/bleu/tokenizer_13a.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/.cache/huggingface/bleu/tokenizer_13a.py -------------------------------------------------------------------------------- /src/.cache/huggingface/rouge/rouge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/.cache/huggingface/rouge/rouge.py -------------------------------------------------------------------------------- /src/configs/__pycache__/config.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/configs/__pycache__/config.cpython-310.pyc -------------------------------------------------------------------------------- /src/configs/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/configs/config.py -------------------------------------------------------------------------------- /src/core/analyst.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/core/analyst.py -------------------------------------------------------------------------------- /src/datasets/__pycache__/base.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/datasets/__pycache__/base.cpython-310.pyc -------------------------------------------------------------------------------- /src/datasets/__pycache__/xinhua.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/datasets/__pycache__/xinhua.cpython-310.pyc -------------------------------------------------------------------------------- /src/datasets/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/datasets/base.py -------------------------------------------------------------------------------- /src/datasets/xinhua.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/datasets/xinhua.py -------------------------------------------------------------------------------- /src/embeddings/__pycache__/base.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/embeddings/__pycache__/base.cpython-310.pyc -------------------------------------------------------------------------------- /src/embeddings/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/embeddings/base.py -------------------------------------------------------------------------------- /src/eval_retrieval/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import BaseRetriever 2 | -------------------------------------------------------------------------------- /src/eval_retrieval/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/eval_retrieval/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /src/eval_retrieval/__pycache__/base.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/eval_retrieval/__pycache__/base.cpython-310.pyc -------------------------------------------------------------------------------- /src/eval_retrieval/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/eval_retrieval/base.py -------------------------------------------------------------------------------- /src/llms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/llms/__init__.py -------------------------------------------------------------------------------- /src/llms/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/llms/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /src/llms/__pycache__/base.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/llms/__pycache__/base.cpython-310.pyc -------------------------------------------------------------------------------- /src/llms/__pycache__/local_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/llms/__pycache__/local_model.cpython-310.pyc -------------------------------------------------------------------------------- /src/llms/__pycache__/remote_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/llms/__pycache__/remote_model.cpython-310.pyc -------------------------------------------------------------------------------- /src/llms/api_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/llms/api_model.py -------------------------------------------------------------------------------- /src/llms/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/llms/base.py -------------------------------------------------------------------------------- /src/llms/local_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/llms/local_model.py -------------------------------------------------------------------------------- /src/llms/remote_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/llms/remote_model.py -------------------------------------------------------------------------------- /src/metric/__pycache__/common.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/metric/__pycache__/common.cpython-310.pyc -------------------------------------------------------------------------------- /src/metric/__pycache__/quest_eval.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/metric/__pycache__/quest_eval.cpython-310.pyc -------------------------------------------------------------------------------- /src/metric/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/metric/common.py -------------------------------------------------------------------------------- /src/metric/quest_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/metric/quest_eval.py -------------------------------------------------------------------------------- /src/prompts/continue_writing.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/prompts/continue_writing.txt -------------------------------------------------------------------------------- /src/prompts/hallu_mod.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/prompts/hallu_mod.txt -------------------------------------------------------------------------------- /src/prompts/quest_answer.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/prompts/quest_answer.txt -------------------------------------------------------------------------------- /src/prompts/quest_eval_answer.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/prompts/quest_eval_answer.txt -------------------------------------------------------------------------------- /src/prompts/quest_eval_gen.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/prompts/quest_eval_gen.txt -------------------------------------------------------------------------------- /src/prompts/summary.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/prompts/summary.txt -------------------------------------------------------------------------------- /src/quest_eval/ContinueWriting_quest_gt_save.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/quest_eval/ContinueWriting_quest_gt_save.json -------------------------------------------------------------------------------- /src/quest_eval/HalluModified_quest_gt_save.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/quest_eval/HalluModified_quest_gt_save.json -------------------------------------------------------------------------------- /src/quest_eval/QuestAnswer1Doc_quest_gt_save.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/quest_eval/QuestAnswer1Doc_quest_gt_save.json -------------------------------------------------------------------------------- /src/quest_eval/QuestAnswer2Docs_quest_gt_save.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/quest_eval/QuestAnswer2Docs_quest_gt_save.json -------------------------------------------------------------------------------- /src/quest_eval/QuestAnswer3Docs_quest_gt_save.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/quest_eval/QuestAnswer3Docs_quest_gt_save.json -------------------------------------------------------------------------------- /src/quest_eval/Summary_quest_gt_save.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/quest_eval/Summary_quest_gt_save.json -------------------------------------------------------------------------------- /src/retrievers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/retrievers/__init__.py -------------------------------------------------------------------------------- /src/retrievers/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/retrievers/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /src/retrievers/__pycache__/base.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/retrievers/__pycache__/base.cpython-310.pyc -------------------------------------------------------------------------------- /src/retrievers/__pycache__/bm25.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/retrievers/__pycache__/bm25.cpython-310.pyc -------------------------------------------------------------------------------- /src/retrievers/__pycache__/hybrid.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/retrievers/__pycache__/hybrid.cpython-310.pyc -------------------------------------------------------------------------------- /src/retrievers/__pycache__/hybrid_rerank.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/retrievers/__pycache__/hybrid_rerank.cpython-310.pyc -------------------------------------------------------------------------------- /src/retrievers/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/retrievers/base.py -------------------------------------------------------------------------------- /src/retrievers/base1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/retrievers/base1.py -------------------------------------------------------------------------------- /src/retrievers/bm25.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/retrievers/bm25.py -------------------------------------------------------------------------------- /src/retrievers/hybrid.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/retrievers/hybrid.py -------------------------------------------------------------------------------- /src/retrievers/hybrid_rerank.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/retrievers/hybrid_rerank.py -------------------------------------------------------------------------------- /src/tasks/__pycache__/base.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/tasks/__pycache__/base.cpython-310.pyc -------------------------------------------------------------------------------- /src/tasks/__pycache__/continue_writing.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/tasks/__pycache__/continue_writing.cpython-310.pyc -------------------------------------------------------------------------------- /src/tasks/__pycache__/hallucinated_modified.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/tasks/__pycache__/hallucinated_modified.cpython-310.pyc -------------------------------------------------------------------------------- /src/tasks/__pycache__/quest_answer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/tasks/__pycache__/quest_answer.cpython-310.pyc -------------------------------------------------------------------------------- /src/tasks/__pycache__/summary.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/tasks/__pycache__/summary.cpython-310.pyc -------------------------------------------------------------------------------- /src/tasks/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/tasks/base.py -------------------------------------------------------------------------------- /src/tasks/continue_writing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/tasks/continue_writing.py -------------------------------------------------------------------------------- /src/tasks/hallucinated_modified.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/tasks/hallucinated_modified.py -------------------------------------------------------------------------------- /src/tasks/quest_answer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/tasks/quest_answer.py -------------------------------------------------------------------------------- /src/tasks/summary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MemTensor/MoM/HEAD/src/tasks/summary.py --------------------------------------------------------------------------------