├── .gitignore ├── construct_embeddings ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-39.pyc │ └── construct_embedding.cpython-39.pyc ├── combine_ndlr_ndgr_rules.py └── construct_embedding.py ├── data ├── exam_questions │ ├── test_questions_human_annotated_basic_morality.csv │ ├── test_questions_human_annotated_law.csv │ ├── test_questions_human_annotated_social_morality.csv │ ├── test_questions_machine_generated_law.csv │ └── test_questions_machine_generated_professional_morality.csv ├── example_data │ ├── examples_law.csv │ └── examples_morality.csv ├── retrieval_location_info │ └── projection_province2file.json └── sampled_data_for_question_generation │ ├── extracted_law_data_for_question_generation.jsonl │ └── extracted_morality_data_for_question_generation.jsonl ├── eval.py ├── evaluation ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-39.pyc │ ├── base_evaluator.cpython-39.pyc │ ├── evaluation_prompt.cpython-39.pyc │ ├── gpt_evaluation.cpython-39.pyc │ └── weight_model_evaluation.cpython-39.pyc ├── base_evaluator.py ├── evaluation_prompt.py ├── gpt_evaluation.py └── weight_model_evaluation.py ├── exam_generation ├── __pycache__ │ ├── eval_generated_question_quality_prompt.cpython-39.pyc │ └── question_generation_prompt.cpython-39.pyc ├── eval_generated_question_quality.py ├── eval_generated_question_quality_prompt.py ├── exam_generation_step.md ├── question_generation_prompt.py ├── question_genration_run.py ├── rule_quality.py └── select_principles_for_question_generation.py ├── figures ├── OPO_framework.png ├── analysis_collected_rules.png └── evaluation_dataset.png ├── readme.md ├── requirements.txt ├── retrieval ├── __pycache__ │ └── retrieval_relevant_docs.cpython-39.pyc └── retrieval_relevant_docs.py └── script ├── __init__.py ├── __pycache__ ├── __init__.cpython-39.pyc ├── calculate_length.cpython-39.pyc ├── city2province.cpython-39.pyc ├── claude_usage.cpython-39.pyc ├── gpt_usage.cpython-39.pyc └── project_province_file.cpython-39.pyc ├── calculate_length.py ├── check_questions.py ├── city2province.py ├── convert_file2csv.py ├── gpt_usage.py └── project_province_file.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/.gitignore -------------------------------------------------------------------------------- /construct_embeddings/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /construct_embeddings/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/construct_embeddings/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /construct_embeddings/__pycache__/construct_embedding.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/construct_embeddings/__pycache__/construct_embedding.cpython-39.pyc -------------------------------------------------------------------------------- /construct_embeddings/combine_ndlr_ndgr_rules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/construct_embeddings/combine_ndlr_ndgr_rules.py -------------------------------------------------------------------------------- /construct_embeddings/construct_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/construct_embeddings/construct_embedding.py -------------------------------------------------------------------------------- /data/exam_questions/test_questions_human_annotated_basic_morality.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/data/exam_questions/test_questions_human_annotated_basic_morality.csv -------------------------------------------------------------------------------- /data/exam_questions/test_questions_human_annotated_law.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/data/exam_questions/test_questions_human_annotated_law.csv -------------------------------------------------------------------------------- /data/exam_questions/test_questions_human_annotated_social_morality.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/data/exam_questions/test_questions_human_annotated_social_morality.csv -------------------------------------------------------------------------------- /data/exam_questions/test_questions_machine_generated_law.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/data/exam_questions/test_questions_machine_generated_law.csv -------------------------------------------------------------------------------- /data/exam_questions/test_questions_machine_generated_professional_morality.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/data/exam_questions/test_questions_machine_generated_professional_morality.csv -------------------------------------------------------------------------------- /data/example_data/examples_law.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/data/example_data/examples_law.csv -------------------------------------------------------------------------------- /data/example_data/examples_morality.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/data/example_data/examples_morality.csv -------------------------------------------------------------------------------- /data/retrieval_location_info/projection_province2file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/data/retrieval_location_info/projection_province2file.json -------------------------------------------------------------------------------- /data/sampled_data_for_question_generation/extracted_law_data_for_question_generation.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/data/sampled_data_for_question_generation/extracted_law_data_for_question_generation.jsonl -------------------------------------------------------------------------------- /data/sampled_data_for_question_generation/extracted_morality_data_for_question_generation.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/data/sampled_data_for_question_generation/extracted_morality_data_for_question_generation.jsonl -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/eval.py -------------------------------------------------------------------------------- /evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /evaluation/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/evaluation/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /evaluation/__pycache__/base_evaluator.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/evaluation/__pycache__/base_evaluator.cpython-39.pyc -------------------------------------------------------------------------------- /evaluation/__pycache__/evaluation_prompt.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/evaluation/__pycache__/evaluation_prompt.cpython-39.pyc -------------------------------------------------------------------------------- /evaluation/__pycache__/gpt_evaluation.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/evaluation/__pycache__/gpt_evaluation.cpython-39.pyc -------------------------------------------------------------------------------- /evaluation/__pycache__/weight_model_evaluation.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/evaluation/__pycache__/weight_model_evaluation.cpython-39.pyc -------------------------------------------------------------------------------- /evaluation/base_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/evaluation/base_evaluator.py -------------------------------------------------------------------------------- /evaluation/evaluation_prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/evaluation/evaluation_prompt.py -------------------------------------------------------------------------------- /evaluation/gpt_evaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/evaluation/gpt_evaluation.py -------------------------------------------------------------------------------- /evaluation/weight_model_evaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/evaluation/weight_model_evaluation.py -------------------------------------------------------------------------------- /exam_generation/__pycache__/eval_generated_question_quality_prompt.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/exam_generation/__pycache__/eval_generated_question_quality_prompt.cpython-39.pyc -------------------------------------------------------------------------------- /exam_generation/__pycache__/question_generation_prompt.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/exam_generation/__pycache__/question_generation_prompt.cpython-39.pyc -------------------------------------------------------------------------------- /exam_generation/eval_generated_question_quality.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/exam_generation/eval_generated_question_quality.py -------------------------------------------------------------------------------- /exam_generation/eval_generated_question_quality_prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/exam_generation/eval_generated_question_quality_prompt.py -------------------------------------------------------------------------------- /exam_generation/exam_generation_step.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/exam_generation/exam_generation_step.md -------------------------------------------------------------------------------- /exam_generation/question_generation_prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/exam_generation/question_generation_prompt.py -------------------------------------------------------------------------------- /exam_generation/question_genration_run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/exam_generation/question_genration_run.py -------------------------------------------------------------------------------- /exam_generation/rule_quality.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/exam_generation/rule_quality.py -------------------------------------------------------------------------------- /exam_generation/select_principles_for_question_generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/exam_generation/select_principles_for_question_generation.py -------------------------------------------------------------------------------- /figures/OPO_framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/figures/OPO_framework.png -------------------------------------------------------------------------------- /figures/analysis_collected_rules.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/figures/analysis_collected_rules.png -------------------------------------------------------------------------------- /figures/evaluation_dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/figures/evaluation_dataset.png -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/readme.md -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/requirements.txt -------------------------------------------------------------------------------- /retrieval/__pycache__/retrieval_relevant_docs.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/retrieval/__pycache__/retrieval_relevant_docs.cpython-39.pyc -------------------------------------------------------------------------------- /retrieval/retrieval_relevant_docs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/retrieval/retrieval_relevant_docs.py -------------------------------------------------------------------------------- /script/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /script/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /script/__pycache__/calculate_length.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/__pycache__/calculate_length.cpython-39.pyc -------------------------------------------------------------------------------- /script/__pycache__/city2province.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/__pycache__/city2province.cpython-39.pyc -------------------------------------------------------------------------------- /script/__pycache__/claude_usage.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/__pycache__/claude_usage.cpython-39.pyc -------------------------------------------------------------------------------- /script/__pycache__/gpt_usage.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/__pycache__/gpt_usage.cpython-39.pyc -------------------------------------------------------------------------------- /script/__pycache__/project_province_file.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/__pycache__/project_province_file.cpython-39.pyc -------------------------------------------------------------------------------- /script/calculate_length.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/calculate_length.py -------------------------------------------------------------------------------- /script/check_questions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/check_questions.py -------------------------------------------------------------------------------- /script/city2province.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/city2province.py -------------------------------------------------------------------------------- /script/convert_file2csv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/convert_file2csv.py -------------------------------------------------------------------------------- /script/gpt_usage.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/gpt_usage.py -------------------------------------------------------------------------------- /script/project_province_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GAIR-NLP/OPO/HEAD/script/project_province_file.py --------------------------------------------------------------------------------