├── CONDA.md ├── LICENSE ├── README.md ├── data ├── judge_prompts.jsonl ├── mt_bench │ ├── misc │ │ └── radar.png │ ├── question.jsonl │ └── reference_answer │ │ ├── gpt-4.jsonl │ │ └── gpt-4o-2024-05-13.jsonl └── vicuna_bench │ ├── question.jsonl │ └── reference_answer │ └── gpt-4.jsonl ├── environment.yaml ├── eval ├── common.py ├── gen_judgment.py ├── gen_model_answer.py ├── math_normalization.py └── show_result.py ├── eval_mdm.sh ├── evaluate_ar.py ├── evaluate_diff.py ├── evaluate_fineweb.py ├── evaluate_gsm8k.py ├── evaluate_reverse.py ├── imgs ├── scale_loss.png └── scale_para.png ├── lit_gpt ├── __init__.py ├── adapter.py ├── adapter_v2.py ├── config.py ├── diffmodel.py ├── fused_cross_entropy.py ├── fused_rotary_embedding.py ├── lora.py ├── model.py ├── model_cache.py ├── packed_dataset.py ├── rmsnorm.py ├── speed_monitor.py ├── tokenizer.py └── utils.py ├── pretrain ├── train_ar.py ├── train_mdm.py └── train_mdm_rl.py ├── scripts └── prepare_fineweb.py └── sft ├── finetune_ar.py ├── finetune_mdm.py ├── finetune_mdm_gsm8k.py ├── finetune_mdm_reverse.py ├── gsm8k_data.py ├── reverse_data.py └── sharegpt_data.py /CONDA.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/CONDA.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/README.md -------------------------------------------------------------------------------- /data/judge_prompts.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/data/judge_prompts.jsonl -------------------------------------------------------------------------------- /data/mt_bench/misc/radar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/data/mt_bench/misc/radar.png -------------------------------------------------------------------------------- /data/mt_bench/question.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/data/mt_bench/question.jsonl -------------------------------------------------------------------------------- /data/mt_bench/reference_answer/gpt-4.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/data/mt_bench/reference_answer/gpt-4.jsonl -------------------------------------------------------------------------------- /data/mt_bench/reference_answer/gpt-4o-2024-05-13.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/data/mt_bench/reference_answer/gpt-4o-2024-05-13.jsonl -------------------------------------------------------------------------------- /data/vicuna_bench/question.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/data/vicuna_bench/question.jsonl -------------------------------------------------------------------------------- /data/vicuna_bench/reference_answer/gpt-4.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/data/vicuna_bench/reference_answer/gpt-4.jsonl -------------------------------------------------------------------------------- /environment.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/environment.yaml -------------------------------------------------------------------------------- /eval/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/eval/common.py -------------------------------------------------------------------------------- /eval/gen_judgment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/eval/gen_judgment.py -------------------------------------------------------------------------------- /eval/gen_model_answer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/eval/gen_model_answer.py -------------------------------------------------------------------------------- /eval/math_normalization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/eval/math_normalization.py -------------------------------------------------------------------------------- /eval/show_result.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/eval/show_result.py -------------------------------------------------------------------------------- /eval_mdm.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/eval_mdm.sh -------------------------------------------------------------------------------- /evaluate_ar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/evaluate_ar.py -------------------------------------------------------------------------------- /evaluate_diff.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/evaluate_diff.py -------------------------------------------------------------------------------- /evaluate_fineweb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/evaluate_fineweb.py -------------------------------------------------------------------------------- /evaluate_gsm8k.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/evaluate_gsm8k.py -------------------------------------------------------------------------------- /evaluate_reverse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/evaluate_reverse.py -------------------------------------------------------------------------------- /imgs/scale_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/imgs/scale_loss.png -------------------------------------------------------------------------------- /imgs/scale_para.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/imgs/scale_para.png -------------------------------------------------------------------------------- /lit_gpt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/__init__.py -------------------------------------------------------------------------------- /lit_gpt/adapter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/adapter.py -------------------------------------------------------------------------------- /lit_gpt/adapter_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/adapter_v2.py -------------------------------------------------------------------------------- /lit_gpt/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/config.py -------------------------------------------------------------------------------- /lit_gpt/diffmodel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/diffmodel.py -------------------------------------------------------------------------------- /lit_gpt/fused_cross_entropy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/fused_cross_entropy.py -------------------------------------------------------------------------------- /lit_gpt/fused_rotary_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/fused_rotary_embedding.py -------------------------------------------------------------------------------- /lit_gpt/lora.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/lora.py -------------------------------------------------------------------------------- /lit_gpt/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/model.py -------------------------------------------------------------------------------- /lit_gpt/model_cache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/model_cache.py -------------------------------------------------------------------------------- /lit_gpt/packed_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/packed_dataset.py -------------------------------------------------------------------------------- /lit_gpt/rmsnorm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/rmsnorm.py -------------------------------------------------------------------------------- /lit_gpt/speed_monitor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/speed_monitor.py -------------------------------------------------------------------------------- /lit_gpt/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/tokenizer.py -------------------------------------------------------------------------------- /lit_gpt/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/lit_gpt/utils.py -------------------------------------------------------------------------------- /pretrain/train_ar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/pretrain/train_ar.py -------------------------------------------------------------------------------- /pretrain/train_mdm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/pretrain/train_mdm.py -------------------------------------------------------------------------------- /pretrain/train_mdm_rl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/pretrain/train_mdm_rl.py -------------------------------------------------------------------------------- /scripts/prepare_fineweb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/scripts/prepare_fineweb.py -------------------------------------------------------------------------------- /sft/finetune_ar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/sft/finetune_ar.py -------------------------------------------------------------------------------- /sft/finetune_mdm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/sft/finetune_mdm.py -------------------------------------------------------------------------------- /sft/finetune_mdm_gsm8k.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/sft/finetune_mdm_gsm8k.py -------------------------------------------------------------------------------- /sft/finetune_mdm_reverse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/sft/finetune_mdm_reverse.py -------------------------------------------------------------------------------- /sft/gsm8k_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/sft/gsm8k_data.py -------------------------------------------------------------------------------- /sft/reverse_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/sft/reverse_data.py -------------------------------------------------------------------------------- /sft/sharegpt_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ML-GSAI/SMDM/HEAD/sft/sharegpt_data.py --------------------------------------------------------------------------------