├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── assets └── codefavor.png ├── codefavor ├── __init__.py ├── data │ ├── __init__.py │ ├── commitpackft.py │ ├── editpackft.py │ └── utility.py ├── evaluate.py ├── merge_model.py ├── prompt │ ├── __init__.py │ ├── commit_instruct.py │ ├── critic_evol.py │ └── utility.py ├── provider │ ├── __init__.py │ ├── base.py │ ├── bedrock.py │ ├── google.py │ ├── hf.py │ └── openai.py ├── template.py └── utility.py ├── datasets └── eval │ ├── LICENSE │ ├── README.md │ ├── correctness-alignment.jsonl.gz │ ├── efficiency-alignment.jsonl.gz │ ├── human-preference-alignment.jsonl.gz │ └── security-alignment.jsonl.gz ├── requirements.txt └── scripts ├── axolotl ├── prepare_data.py ├── prepare_model.py ├── recipe │ ├── .gitignore │ ├── controlled │ │ ├── mistral │ │ │ ├── cls-critic-evol-from-llama3-70b-70b.yaml │ │ │ ├── cls-critic-evol-from-llama3-8b-8b.yaml │ │ │ ├── gen-cls-mix-from-llama3-70b.yaml │ │ │ └── withcmt-cls-mix-from-llama3-70b.yaml │ │ └── nemo │ │ │ ├── cls-critic-evol-from-llama3-70b-70b.yaml │ │ │ ├── cls-critic-evol-from-llama3-8b-8b.yaml │ │ │ ├── gen-mix-from-llama3-70b.yaml │ │ │ └── withcmt-cls-mix-from-llama3-70b.yaml │ ├── gemma │ │ ├── cls-commit-instruct-from-llama3-70b.yaml │ │ ├── cls-critic-evol-from-llama3-70b.yaml │ │ ├── cls-mix-from-llama3-70b.yaml │ │ ├── gen-ce-from-llama3-70b.yaml │ │ ├── gen-ci-from-llama3-70b.yaml │ │ └── gen-mix-from-llama3-70b.yaml │ ├── llama │ │ ├── gen-ci-from-llama3-70b.yaml │ │ ├── gen-critic-evol-from-llama3-70b.yaml │ │ ├── gen-mix-from-llama3-70b.yaml │ │ ├── llama3-8b-cls-commit-instruct-from-llama3-70b.yaml │ │ ├── llama3-8b-cls-critic-evol-from-llama3-70b.yaml │ │ └── llama3-8b-cls-mix-from-llama3-70b.yaml │ ├── mistral │ │ ├── gen-ci-from-llama3-70b.yaml │ │ ├── gen-critic-evol-from-llama3-70b.yaml │ │ ├── gen-mix-from-llama3-70b.yaml │ │ ├── mistral-7b-cls-commit-instruct-from-llama3-70b.yaml │ │ ├── mistral-7b-cls-critic-evol-from-llama3-70b.yaml │ │ └── mistral-7b-cls-mix-from-llama3-70b.yaml │ └── nemo │ │ ├── cls-commit-instruct-from-llama3-70b.yaml │ │ ├── cls-critic-evol-from-llama3-70b.yaml │ │ ├── cls-mix-from-llama3-70b.yaml │ │ ├── gen-ci-from-llama3-70b.yaml │ │ ├── gen-critic-evol-from-llama3-70b.yaml │ │ └── gen-mix-from-llama3-70b.yaml ├── zero1.json ├── zero2.json └── zero3.json ├── data ├── codegen_soss.py ├── decompose_ce.py └── decompose_ci.py ├── debug ├── commit_instruct_inspect.py ├── critic_evol_inspect.py └── eval_inspect.py └── eval ├── contamination.py └── message_analysis.py /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/LICENSE -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/README.md -------------------------------------------------------------------------------- /assets/codefavor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/assets/codefavor.png -------------------------------------------------------------------------------- /codefavor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/__init__.py -------------------------------------------------------------------------------- /codefavor/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/data/__init__.py -------------------------------------------------------------------------------- /codefavor/data/commitpackft.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/data/commitpackft.py -------------------------------------------------------------------------------- /codefavor/data/editpackft.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/data/editpackft.py -------------------------------------------------------------------------------- /codefavor/data/utility.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/data/utility.py -------------------------------------------------------------------------------- /codefavor/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/evaluate.py -------------------------------------------------------------------------------- /codefavor/merge_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/merge_model.py -------------------------------------------------------------------------------- /codefavor/prompt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/prompt/__init__.py -------------------------------------------------------------------------------- /codefavor/prompt/commit_instruct.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/prompt/commit_instruct.py -------------------------------------------------------------------------------- /codefavor/prompt/critic_evol.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/prompt/critic_evol.py -------------------------------------------------------------------------------- /codefavor/prompt/utility.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/prompt/utility.py -------------------------------------------------------------------------------- /codefavor/provider/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/provider/__init__.py -------------------------------------------------------------------------------- /codefavor/provider/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/provider/base.py -------------------------------------------------------------------------------- /codefavor/provider/bedrock.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/provider/bedrock.py -------------------------------------------------------------------------------- /codefavor/provider/google.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/provider/google.py -------------------------------------------------------------------------------- /codefavor/provider/hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/provider/hf.py -------------------------------------------------------------------------------- /codefavor/provider/openai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/provider/openai.py -------------------------------------------------------------------------------- /codefavor/template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/template.py -------------------------------------------------------------------------------- /codefavor/utility.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/codefavor/utility.py -------------------------------------------------------------------------------- /datasets/eval/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/datasets/eval/LICENSE -------------------------------------------------------------------------------- /datasets/eval/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/datasets/eval/README.md -------------------------------------------------------------------------------- /datasets/eval/correctness-alignment.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/datasets/eval/correctness-alignment.jsonl.gz -------------------------------------------------------------------------------- /datasets/eval/efficiency-alignment.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/datasets/eval/efficiency-alignment.jsonl.gz -------------------------------------------------------------------------------- /datasets/eval/human-preference-alignment.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/datasets/eval/human-preference-alignment.jsonl.gz -------------------------------------------------------------------------------- /datasets/eval/security-alignment.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/datasets/eval/security-alignment.jsonl.gz -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/axolotl/prepare_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/prepare_data.py -------------------------------------------------------------------------------- /scripts/axolotl/prepare_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/prepare_model.py -------------------------------------------------------------------------------- /scripts/axolotl/recipe/.gitignore: -------------------------------------------------------------------------------- 1 | backup/ 2 | test/ 3 | repro/ 4 | -------------------------------------------------------------------------------- /scripts/axolotl/recipe/controlled/mistral/cls-critic-evol-from-llama3-70b-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/controlled/mistral/cls-critic-evol-from-llama3-70b-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/controlled/mistral/cls-critic-evol-from-llama3-8b-8b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/controlled/mistral/cls-critic-evol-from-llama3-8b-8b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/controlled/mistral/gen-cls-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/controlled/mistral/gen-cls-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/controlled/mistral/withcmt-cls-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/controlled/mistral/withcmt-cls-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/controlled/nemo/cls-critic-evol-from-llama3-70b-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/controlled/nemo/cls-critic-evol-from-llama3-70b-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/controlled/nemo/cls-critic-evol-from-llama3-8b-8b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/controlled/nemo/cls-critic-evol-from-llama3-8b-8b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/controlled/nemo/gen-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/controlled/nemo/gen-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/controlled/nemo/withcmt-cls-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/controlled/nemo/withcmt-cls-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/gemma/cls-commit-instruct-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/gemma/cls-commit-instruct-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/gemma/cls-critic-evol-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/gemma/cls-critic-evol-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/gemma/cls-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/gemma/cls-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/gemma/gen-ce-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/gemma/gen-ce-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/gemma/gen-ci-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/gemma/gen-ci-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/gemma/gen-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/gemma/gen-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/llama/gen-ci-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/llama/gen-ci-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/llama/gen-critic-evol-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/llama/gen-critic-evol-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/llama/gen-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/llama/gen-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/llama/llama3-8b-cls-commit-instruct-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/llama/llama3-8b-cls-commit-instruct-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/llama/llama3-8b-cls-critic-evol-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/llama/llama3-8b-cls-critic-evol-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/llama/llama3-8b-cls-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/llama/llama3-8b-cls-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/mistral/gen-ci-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/mistral/gen-ci-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/mistral/gen-critic-evol-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/mistral/gen-critic-evol-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/mistral/gen-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/mistral/gen-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/mistral/mistral-7b-cls-commit-instruct-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/mistral/mistral-7b-cls-commit-instruct-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/mistral/mistral-7b-cls-critic-evol-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/mistral/mistral-7b-cls-critic-evol-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/mistral/mistral-7b-cls-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/mistral/mistral-7b-cls-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/nemo/cls-commit-instruct-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/nemo/cls-commit-instruct-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/nemo/cls-critic-evol-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/nemo/cls-critic-evol-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/nemo/cls-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/nemo/cls-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/nemo/gen-ci-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/nemo/gen-ci-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/nemo/gen-critic-evol-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/nemo/gen-critic-evol-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/recipe/nemo/gen-mix-from-llama3-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/recipe/nemo/gen-mix-from-llama3-70b.yaml -------------------------------------------------------------------------------- /scripts/axolotl/zero1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/zero1.json -------------------------------------------------------------------------------- /scripts/axolotl/zero2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/zero2.json -------------------------------------------------------------------------------- /scripts/axolotl/zero3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/axolotl/zero3.json -------------------------------------------------------------------------------- /scripts/data/codegen_soss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/data/codegen_soss.py -------------------------------------------------------------------------------- /scripts/data/decompose_ce.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/data/decompose_ce.py -------------------------------------------------------------------------------- /scripts/data/decompose_ci.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/data/decompose_ci.py -------------------------------------------------------------------------------- /scripts/debug/commit_instruct_inspect.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/debug/commit_instruct_inspect.py -------------------------------------------------------------------------------- /scripts/debug/critic_evol_inspect.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/debug/critic_evol_inspect.py -------------------------------------------------------------------------------- /scripts/debug/eval_inspect.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/debug/eval_inspect.py -------------------------------------------------------------------------------- /scripts/eval/contamination.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/eval/contamination.py -------------------------------------------------------------------------------- /scripts/eval/message_analysis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/llm-code-preference/HEAD/scripts/eval/message_analysis.py --------------------------------------------------------------------------------