├── LICENSE ├── README.md ├── data └── UltraSafety.jsonl ├── figs ├── baseline.png ├── cpo.png └── results.png ├── requirements ├── requirements_cdpo.txt └── requirements_cpsft.txt ├── scripts ├── dpo_ultrafeedback_cfg.json ├── dpo_ultrasafety_cfg.json ├── run_cdpo_data_preparation.sh ├── run_cpsft.sh ├── run_harmlessness_test.sh └── run_honesty_test.sh └── src ├── CDPO ├── cdpo │ ├── cdpo_general.py │ ├── cdpo_ultrafeedback.py │ └── cdpo_ultrasafety.py └── data_preparation │ ├── data_dpo_ultrafeedback.py │ ├── data_dpo_ultrasafety.py │ ├── dpo_UltraSafety.json │ ├── lib │ ├── __pycache__ │ │ ├── file_func.cpython-310.pyc │ │ └── file_func.cpython-39.pyc │ └── file_func.py │ └── main.py └── CPSFT ├── cpsft ├── deepspeed_config │ ├── ZeRO_0.json │ ├── ZeRO_1.json │ ├── ZeRO_2.json │ ├── ZeRO_3.json │ ├── ZeRO_3_cpu.json │ └── deepspeed_zero3.yaml ├── templates │ ├── alpaca.json │ ├── alpaca_legacy.json │ ├── alpaca_short.json │ ├── llama.json │ ├── mistral.json │ ├── mistral_delete.json │ ├── starchat.json │ ├── ultralm.json │ ├── vigogne.json │ ├── wizard.json │ └── zephyr.json ├── test_data_harmlessness │ ├── harmful_bench_0.json │ └── harmful_bench_1.json ├── test_data_honesty │ ├── 1.json │ ├── 2.json │ ├── 3.json │ ├── 4.json │ ├── 5.json │ └── HaluEval.json ├── test_harmlessness.py ├── test_honesty.py ├── train_sft.py └── utils │ └── prompter.py └── data_preparation └── data_preparation_cpsft.py /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/README.md -------------------------------------------------------------------------------- /data/UltraSafety.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/data/UltraSafety.jsonl -------------------------------------------------------------------------------- /figs/baseline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/figs/baseline.png -------------------------------------------------------------------------------- /figs/cpo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/figs/cpo.png -------------------------------------------------------------------------------- /figs/results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/figs/results.png -------------------------------------------------------------------------------- /requirements/requirements_cdpo.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/requirements/requirements_cdpo.txt -------------------------------------------------------------------------------- /requirements/requirements_cpsft.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/requirements/requirements_cpsft.txt -------------------------------------------------------------------------------- /scripts/dpo_ultrafeedback_cfg.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/scripts/dpo_ultrafeedback_cfg.json -------------------------------------------------------------------------------- /scripts/dpo_ultrasafety_cfg.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/scripts/dpo_ultrasafety_cfg.json -------------------------------------------------------------------------------- /scripts/run_cdpo_data_preparation.sh: -------------------------------------------------------------------------------- 1 | cd ../src/CDPO/data_preparation 2 | python ./main.py -------------------------------------------------------------------------------- /scripts/run_cpsft.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/scripts/run_cpsft.sh -------------------------------------------------------------------------------- /scripts/run_harmlessness_test.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/scripts/run_harmlessness_test.sh -------------------------------------------------------------------------------- /scripts/run_honesty_test.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/scripts/run_honesty_test.sh -------------------------------------------------------------------------------- /src/CDPO/cdpo/cdpo_general.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CDPO/cdpo/cdpo_general.py -------------------------------------------------------------------------------- /src/CDPO/cdpo/cdpo_ultrafeedback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CDPO/cdpo/cdpo_ultrafeedback.py -------------------------------------------------------------------------------- /src/CDPO/cdpo/cdpo_ultrasafety.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CDPO/cdpo/cdpo_ultrasafety.py -------------------------------------------------------------------------------- /src/CDPO/data_preparation/data_dpo_ultrafeedback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CDPO/data_preparation/data_dpo_ultrafeedback.py -------------------------------------------------------------------------------- /src/CDPO/data_preparation/data_dpo_ultrasafety.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CDPO/data_preparation/data_dpo_ultrasafety.py -------------------------------------------------------------------------------- /src/CDPO/data_preparation/dpo_UltraSafety.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CDPO/data_preparation/dpo_UltraSafety.json -------------------------------------------------------------------------------- /src/CDPO/data_preparation/lib/__pycache__/file_func.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CDPO/data_preparation/lib/__pycache__/file_func.cpython-310.pyc -------------------------------------------------------------------------------- /src/CDPO/data_preparation/lib/__pycache__/file_func.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CDPO/data_preparation/lib/__pycache__/file_func.cpython-39.pyc -------------------------------------------------------------------------------- /src/CDPO/data_preparation/lib/file_func.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CDPO/data_preparation/lib/file_func.py -------------------------------------------------------------------------------- /src/CDPO/data_preparation/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CDPO/data_preparation/main.py -------------------------------------------------------------------------------- /src/CPSFT/cpsft/deepspeed_config/ZeRO_0.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/deepspeed_config/ZeRO_0.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/deepspeed_config/ZeRO_1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/deepspeed_config/ZeRO_1.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/deepspeed_config/ZeRO_2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/deepspeed_config/ZeRO_2.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/deepspeed_config/ZeRO_3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/deepspeed_config/ZeRO_3.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/deepspeed_config/ZeRO_3_cpu.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/deepspeed_config/ZeRO_3_cpu.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/deepspeed_config/deepspeed_zero3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/deepspeed_config/deepspeed_zero3.yaml -------------------------------------------------------------------------------- /src/CPSFT/cpsft/templates/alpaca.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/templates/alpaca.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/templates/alpaca_legacy.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/templates/alpaca_legacy.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/templates/alpaca_short.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/templates/alpaca_short.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/templates/llama.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/templates/llama.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/templates/mistral.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/templates/mistral.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/templates/mistral_delete.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/templates/mistral_delete.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/templates/starchat.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/templates/starchat.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/templates/ultralm.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/templates/ultralm.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/templates/vigogne.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/templates/vigogne.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/templates/wizard.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/templates/wizard.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/templates/zephyr.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/templates/zephyr.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/test_data_harmlessness/harmful_bench_0.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/test_data_harmlessness/harmful_bench_0.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/test_data_harmlessness/harmful_bench_1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/test_data_harmlessness/harmful_bench_1.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/test_data_honesty/1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/test_data_honesty/1.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/test_data_honesty/2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/test_data_honesty/2.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/test_data_honesty/3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/test_data_honesty/3.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/test_data_honesty/4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/test_data_honesty/4.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/test_data_honesty/5.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/test_data_honesty/5.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/test_data_honesty/HaluEval.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/test_data_honesty/HaluEval.json -------------------------------------------------------------------------------- /src/CPSFT/cpsft/test_harmlessness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/test_harmlessness.py -------------------------------------------------------------------------------- /src/CPSFT/cpsft/test_honesty.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/test_honesty.py -------------------------------------------------------------------------------- /src/CPSFT/cpsft/train_sft.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/train_sft.py -------------------------------------------------------------------------------- /src/CPSFT/cpsft/utils/prompter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/cpsft/utils/prompter.py -------------------------------------------------------------------------------- /src/CPSFT/data_preparation/data_preparation_cpsft.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/CPO/HEAD/src/CPSFT/data_preparation/data_preparation_cpsft.py --------------------------------------------------------------------------------