├── .gitignore ├── LICENSE ├── README.md ├── ariadne_ai ├── __init__.py ├── evaluators │ ├── __init__.py │ ├── rag │ │ ├── __init__.py │ │ ├── answer_relevance_evaluator.py │ │ ├── context_relevance_evaluator.py │ │ ├── faithfulness_evaluator.py │ │ └── rag_evaluator.py │ └── text_summarization │ │ ├── __init__.py │ │ ├── hallucination_evaluator.py │ │ ├── informativeness_evaluator.py │ │ └── summarization_evaluator.py ├── llms │ ├── __init__.py │ ├── base_llm_evaluator.py │ ├── open_ai_completion.py │ ├── rag │ │ ├── __init__.py │ │ ├── answer_relevance.py │ │ ├── context_relevance.py │ │ └── faithfulness.py │ └── text_summarization │ │ ├── __init__.py │ │ ├── question_answerer.py │ │ └── question_generator.py ├── loaders │ ├── __init__.py │ ├── loader.py │ ├── rag_loader.py │ └── summarization_loader.py ├── metrics │ ├── __init__.py │ ├── metric.py │ ├── rag │ │ ├── __init__.py │ │ ├── answer_relevance_failure.py │ │ ├── context_relevance_failure.py │ │ └── faithfulness_failure.py │ └── text_summarization │ │ ├── __init__.py │ │ ├── aggreement_score.py │ │ ├── contradiction_failure.py │ │ ├── hallucination_failure.py │ │ └── informativeness_failure.py └── publishers │ ├── __init__.py │ ├── publisher.py │ └── publisher_log.py ├── data ├── logs │ ├── log_rag_ans_rel_eval.json │ ├── log_rag_cont_rel_eval.json │ ├── log_rag_faith_eval.json │ ├── log_sum_hal_eval_gpt_35_questions_2.json │ ├── log_sum_hal_eval_gpt_35_questions_5.json │ ├── logs_xsum_sample.json │ ├── perf_rag_ans_rel_eval.txt │ ├── perf_rag_cont_rel_eval.txt │ ├── perf_rag_faith_eval.txt │ ├── perf_sum_hal_eval_gpt_35_questions_2.txt │ ├── perf_sum_hal_eval_gpt_35_questions_5.txt │ └── perf_xsum_sample.json ├── rag │ ├── answer_relevance_sample.json │ ├── context_relevance_sample.json │ └── faithfulness_sample.json └── text_summarization │ └── xsum_sample.json ├── docs └── QAG_approach.png ├── examples ├── data │ └── logs │ │ ├── log_rag_faith_eval.json │ │ └── perf_rag_faith_eval.txt ├── detect_rag_hallucinations.ipynb └── detect_text_summarization_hallucinations.ipynb ├── poetry.lock ├── pyproject.toml ├── references.md ├── run_experiment_example.py ├── run_experiment_rag.py ├── run_experiment_summarization.py └── test ├── __init__.py └── test_loader.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/README.md -------------------------------------------------------------------------------- /ariadne_ai/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/evaluators/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/evaluators/rag/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/evaluators/rag/answer_relevance_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/evaluators/rag/answer_relevance_evaluator.py -------------------------------------------------------------------------------- /ariadne_ai/evaluators/rag/context_relevance_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/evaluators/rag/context_relevance_evaluator.py -------------------------------------------------------------------------------- /ariadne_ai/evaluators/rag/faithfulness_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/evaluators/rag/faithfulness_evaluator.py -------------------------------------------------------------------------------- /ariadne_ai/evaluators/rag/rag_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/evaluators/rag/rag_evaluator.py -------------------------------------------------------------------------------- /ariadne_ai/evaluators/text_summarization/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/evaluators/text_summarization/hallucination_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/evaluators/text_summarization/hallucination_evaluator.py -------------------------------------------------------------------------------- /ariadne_ai/evaluators/text_summarization/informativeness_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/evaluators/text_summarization/informativeness_evaluator.py -------------------------------------------------------------------------------- /ariadne_ai/evaluators/text_summarization/summarization_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/evaluators/text_summarization/summarization_evaluator.py -------------------------------------------------------------------------------- /ariadne_ai/llms/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/llms/base_llm_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/llms/base_llm_evaluator.py -------------------------------------------------------------------------------- /ariadne_ai/llms/open_ai_completion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/llms/open_ai_completion.py -------------------------------------------------------------------------------- /ariadne_ai/llms/rag/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/llms/rag/answer_relevance.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/llms/rag/answer_relevance.py -------------------------------------------------------------------------------- /ariadne_ai/llms/rag/context_relevance.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/llms/rag/context_relevance.py -------------------------------------------------------------------------------- /ariadne_ai/llms/rag/faithfulness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/llms/rag/faithfulness.py -------------------------------------------------------------------------------- /ariadne_ai/llms/text_summarization/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/llms/text_summarization/question_answerer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/llms/text_summarization/question_answerer.py -------------------------------------------------------------------------------- /ariadne_ai/llms/text_summarization/question_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/llms/text_summarization/question_generator.py -------------------------------------------------------------------------------- /ariadne_ai/loaders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/loaders/loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/loaders/loader.py -------------------------------------------------------------------------------- /ariadne_ai/loaders/rag_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/loaders/rag_loader.py -------------------------------------------------------------------------------- /ariadne_ai/loaders/summarization_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/loaders/summarization_loader.py -------------------------------------------------------------------------------- /ariadne_ai/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/metrics/metric.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/metrics/metric.py -------------------------------------------------------------------------------- /ariadne_ai/metrics/rag/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/metrics/rag/answer_relevance_failure.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/metrics/rag/answer_relevance_failure.py -------------------------------------------------------------------------------- /ariadne_ai/metrics/rag/context_relevance_failure.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/metrics/rag/context_relevance_failure.py -------------------------------------------------------------------------------- /ariadne_ai/metrics/rag/faithfulness_failure.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/metrics/rag/faithfulness_failure.py -------------------------------------------------------------------------------- /ariadne_ai/metrics/text_summarization/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/metrics/text_summarization/aggreement_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/metrics/text_summarization/aggreement_score.py -------------------------------------------------------------------------------- /ariadne_ai/metrics/text_summarization/contradiction_failure.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/metrics/text_summarization/contradiction_failure.py -------------------------------------------------------------------------------- /ariadne_ai/metrics/text_summarization/hallucination_failure.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/metrics/text_summarization/hallucination_failure.py -------------------------------------------------------------------------------- /ariadne_ai/metrics/text_summarization/informativeness_failure.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/metrics/text_summarization/informativeness_failure.py -------------------------------------------------------------------------------- /ariadne_ai/publishers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ariadne_ai/publishers/publisher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/publishers/publisher.py -------------------------------------------------------------------------------- /ariadne_ai/publishers/publisher_log.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/ariadne_ai/publishers/publisher_log.py -------------------------------------------------------------------------------- /data/logs/log_rag_ans_rel_eval.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/log_rag_ans_rel_eval.json -------------------------------------------------------------------------------- /data/logs/log_rag_cont_rel_eval.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/log_rag_cont_rel_eval.json -------------------------------------------------------------------------------- /data/logs/log_rag_faith_eval.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/log_rag_faith_eval.json -------------------------------------------------------------------------------- /data/logs/log_sum_hal_eval_gpt_35_questions_2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/log_sum_hal_eval_gpt_35_questions_2.json -------------------------------------------------------------------------------- /data/logs/log_sum_hal_eval_gpt_35_questions_5.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/log_sum_hal_eval_gpt_35_questions_5.json -------------------------------------------------------------------------------- /data/logs/logs_xsum_sample.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/logs_xsum_sample.json -------------------------------------------------------------------------------- /data/logs/perf_rag_ans_rel_eval.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/perf_rag_ans_rel_eval.txt -------------------------------------------------------------------------------- /data/logs/perf_rag_cont_rel_eval.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/perf_rag_cont_rel_eval.txt -------------------------------------------------------------------------------- /data/logs/perf_rag_faith_eval.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/perf_rag_faith_eval.txt -------------------------------------------------------------------------------- /data/logs/perf_sum_hal_eval_gpt_35_questions_2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/perf_sum_hal_eval_gpt_35_questions_2.txt -------------------------------------------------------------------------------- /data/logs/perf_sum_hal_eval_gpt_35_questions_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/perf_sum_hal_eval_gpt_35_questions_5.txt -------------------------------------------------------------------------------- /data/logs/perf_xsum_sample.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/logs/perf_xsum_sample.json -------------------------------------------------------------------------------- /data/rag/answer_relevance_sample.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/rag/answer_relevance_sample.json -------------------------------------------------------------------------------- /data/rag/context_relevance_sample.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/rag/context_relevance_sample.json -------------------------------------------------------------------------------- /data/rag/faithfulness_sample.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/rag/faithfulness_sample.json -------------------------------------------------------------------------------- /data/text_summarization/xsum_sample.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/data/text_summarization/xsum_sample.json -------------------------------------------------------------------------------- /docs/QAG_approach.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/docs/QAG_approach.png -------------------------------------------------------------------------------- /examples/data/logs/log_rag_faith_eval.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/examples/data/logs/log_rag_faith_eval.json -------------------------------------------------------------------------------- /examples/data/logs/perf_rag_faith_eval.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/examples/data/logs/perf_rag_faith_eval.txt -------------------------------------------------------------------------------- /examples/detect_rag_hallucinations.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/examples/detect_rag_hallucinations.ipynb -------------------------------------------------------------------------------- /examples/detect_text_summarization_hallucinations.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/examples/detect_text_summarization_hallucinations.ipynb -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/poetry.lock -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/pyproject.toml -------------------------------------------------------------------------------- /references.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/references.md -------------------------------------------------------------------------------- /run_experiment_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/run_experiment_example.py -------------------------------------------------------------------------------- /run_experiment_rag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/run_experiment_rag.py -------------------------------------------------------------------------------- /run_experiment_summarization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/run_experiment_summarization.py -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/athina-ai/ariadne/HEAD/test/test_loader.py --------------------------------------------------------------------------------