├── .gitignore ├── .gitmodules ├── AI_ETHICS.md ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── SECURITY.md ├── data ├── example.json ├── example_knowledge.txt └── report_example.json ├── figures └── title.png ├── main.py ├── pyproject.toml ├── requirements.txt └── src └── auditnlg ├── constraint ├── __init__.py ├── constraint_checker.py ├── exam.py └── prompts.py ├── explain.py ├── factualness ├── classifier.py ├── exam.py ├── openAIscorers │ └── constants.py ├── qafacteval_utils │ ├── answer_selection.py │ ├── evaluator.py │ ├── overall_model.py │ ├── qa_model.py │ └── utils.py ├── summac_utils │ └── evaluator.py ├── unieval_utils │ ├── evaluator.py │ └── scorer.py └── utils.py ├── llm_wrapper.py ├── regeneration ├── constitutional_ai │ ├── __init__.py │ ├── constants.py │ └── critique_revise.py ├── modeling.py ├── prompt_expansion │ ├── __init__.py │ ├── constants.py │ └── prompt_with_guidelines.py ├── prompt_helper.py └── self_refine │ ├── __init__.py │ ├── constants.py │ └── pipeline.py ├── safety ├── classifier.py ├── constants.py ├── exam.py ├── perspective.py ├── safety_bench │ └── checker.py ├── safety_generator.py ├── self_diagnosis │ ├── generation.py │ ├── generation_utils.py │ ├── modeling.py │ └── self_diagnosis.py └── sensitive_checker.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | ./output/* 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/.gitmodules -------------------------------------------------------------------------------- /AI_ETHICS.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/AI_ETHICS.md -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/CODEOWNERS -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/LICENSE.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/README.md -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/SECURITY.md -------------------------------------------------------------------------------- /data/example.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/data/example.json -------------------------------------------------------------------------------- /data/example_knowledge.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/data/example_knowledge.txt -------------------------------------------------------------------------------- /data/report_example.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/data/report_example.json -------------------------------------------------------------------------------- /figures/title.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/figures/title.png -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/main.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/pyproject.toml -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/requirements.txt -------------------------------------------------------------------------------- /src/auditnlg/constraint/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/auditnlg/constraint/constraint_checker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/constraint/constraint_checker.py -------------------------------------------------------------------------------- /src/auditnlg/constraint/exam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/constraint/exam.py -------------------------------------------------------------------------------- /src/auditnlg/constraint/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/constraint/prompts.py -------------------------------------------------------------------------------- /src/auditnlg/explain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/explain.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/classifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/classifier.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/exam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/exam.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/openAIscorers/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/openAIscorers/constants.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/qafacteval_utils/answer_selection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/qafacteval_utils/answer_selection.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/qafacteval_utils/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/qafacteval_utils/evaluator.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/qafacteval_utils/overall_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/qafacteval_utils/overall_model.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/qafacteval_utils/qa_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/qafacteval_utils/qa_model.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/qafacteval_utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/qafacteval_utils/utils.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/summac_utils/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/summac_utils/evaluator.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/unieval_utils/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/unieval_utils/evaluator.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/unieval_utils/scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/unieval_utils/scorer.py -------------------------------------------------------------------------------- /src/auditnlg/factualness/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/factualness/utils.py -------------------------------------------------------------------------------- /src/auditnlg/llm_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/llm_wrapper.py -------------------------------------------------------------------------------- /src/auditnlg/regeneration/constitutional_ai/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/auditnlg/regeneration/constitutional_ai/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/regeneration/constitutional_ai/constants.py -------------------------------------------------------------------------------- /src/auditnlg/regeneration/constitutional_ai/critique_revise.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/regeneration/constitutional_ai/critique_revise.py -------------------------------------------------------------------------------- /src/auditnlg/regeneration/modeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/regeneration/modeling.py -------------------------------------------------------------------------------- /src/auditnlg/regeneration/prompt_expansion/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/auditnlg/regeneration/prompt_expansion/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/regeneration/prompt_expansion/constants.py -------------------------------------------------------------------------------- /src/auditnlg/regeneration/prompt_expansion/prompt_with_guidelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/regeneration/prompt_expansion/prompt_with_guidelines.py -------------------------------------------------------------------------------- /src/auditnlg/regeneration/prompt_helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/regeneration/prompt_helper.py -------------------------------------------------------------------------------- /src/auditnlg/regeneration/self_refine/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/auditnlg/regeneration/self_refine/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/regeneration/self_refine/constants.py -------------------------------------------------------------------------------- /src/auditnlg/regeneration/self_refine/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/regeneration/self_refine/pipeline.py -------------------------------------------------------------------------------- /src/auditnlg/safety/classifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/safety/classifier.py -------------------------------------------------------------------------------- /src/auditnlg/safety/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/safety/constants.py -------------------------------------------------------------------------------- /src/auditnlg/safety/exam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/safety/exam.py -------------------------------------------------------------------------------- /src/auditnlg/safety/perspective.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/safety/perspective.py -------------------------------------------------------------------------------- /src/auditnlg/safety/safety_bench/checker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/safety/safety_bench/checker.py -------------------------------------------------------------------------------- /src/auditnlg/safety/safety_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/safety/safety_generator.py -------------------------------------------------------------------------------- /src/auditnlg/safety/self_diagnosis/generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/safety/self_diagnosis/generation.py -------------------------------------------------------------------------------- /src/auditnlg/safety/self_diagnosis/generation_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/safety/self_diagnosis/generation_utils.py -------------------------------------------------------------------------------- /src/auditnlg/safety/self_diagnosis/modeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/safety/self_diagnosis/modeling.py -------------------------------------------------------------------------------- /src/auditnlg/safety/self_diagnosis/self_diagnosis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/safety/self_diagnosis/self_diagnosis.py -------------------------------------------------------------------------------- /src/auditnlg/safety/sensitive_checker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/safety/sensitive_checker.py -------------------------------------------------------------------------------- /src/auditnlg/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/AuditNLG/HEAD/src/auditnlg/utils.py --------------------------------------------------------------------------------