├── .gitignore ├── LICENSE ├── README.md ├── README_zh.md ├── assets ├── eng1.png ├── eng2.png ├── eng3.png ├── image1.png ├── image2.png ├── image3.png ├── instruct_loss.png ├── multiturn_chat.jpeg ├── multiturn_chat_en.jpeg └── pretrain_loss.png ├── chat_server.py ├── configs ├── accelerate_configs │ ├── ds_stage1.yaml │ ├── ds_stage2.yaml │ ├── ds_stage3.yaml │ └── ds_stage3_offload.yaml ├── instruct_config.yaml ├── model_configs │ ├── 13B.json │ ├── 33B.json │ ├── 65B.json │ └── 7B.json ├── pretrain_config.yaml └── tokenizer_models │ ├── 10w_vocab_wudao5_pile10.model │ ├── 4w_cn_vocab_wudao15.model │ ├── llama_tokenizer.model │ └── llama_tokenizer_extended.model ├── data ├── download_instruct.sh ├── download_the_pile.sh ├── download_wudao.sh ├── preprocess_instruction.py ├── preprocess_the_pile.py └── preprocess_wudao.py ├── dataset ├── dataset.py └── validation.py ├── requirements.txt ├── solver └── trainer.py ├── train_lm.py └── utils ├── convert_ckpt.py ├── merge_tokenizer.py ├── speed_test ├── accelerate │ ├── ddp.yaml │ ├── deepspeed_stage1.yaml │ ├── deepspeed_stage2.yaml │ ├── deepspeed_stage3.yaml │ ├── deepspeed_stage3_dynamo.yaml │ ├── deepspeed_stage3_offload.yaml │ ├── fsdp.yaml │ ├── megatron.yaml │ ├── run.py │ └── run.sh ├── colossal-ai │ ├── run.py │ ├── run.sh │ └── utils.py └── lightning │ └── run.py └── train_tokenizer.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/README.md -------------------------------------------------------------------------------- /README_zh.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/README_zh.md -------------------------------------------------------------------------------- /assets/eng1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/assets/eng1.png -------------------------------------------------------------------------------- /assets/eng2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/assets/eng2.png -------------------------------------------------------------------------------- /assets/eng3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/assets/eng3.png -------------------------------------------------------------------------------- /assets/image1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/assets/image1.png -------------------------------------------------------------------------------- /assets/image2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/assets/image2.png -------------------------------------------------------------------------------- /assets/image3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/assets/image3.png -------------------------------------------------------------------------------- /assets/instruct_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/assets/instruct_loss.png -------------------------------------------------------------------------------- /assets/multiturn_chat.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/assets/multiturn_chat.jpeg -------------------------------------------------------------------------------- /assets/multiturn_chat_en.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/assets/multiturn_chat_en.jpeg -------------------------------------------------------------------------------- /assets/pretrain_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/assets/pretrain_loss.png -------------------------------------------------------------------------------- /chat_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/chat_server.py -------------------------------------------------------------------------------- /configs/accelerate_configs/ds_stage1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/accelerate_configs/ds_stage1.yaml -------------------------------------------------------------------------------- /configs/accelerate_configs/ds_stage2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/accelerate_configs/ds_stage2.yaml -------------------------------------------------------------------------------- /configs/accelerate_configs/ds_stage3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/accelerate_configs/ds_stage3.yaml -------------------------------------------------------------------------------- /configs/accelerate_configs/ds_stage3_offload.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/accelerate_configs/ds_stage3_offload.yaml -------------------------------------------------------------------------------- /configs/instruct_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/instruct_config.yaml -------------------------------------------------------------------------------- /configs/model_configs/13B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/model_configs/13B.json -------------------------------------------------------------------------------- /configs/model_configs/33B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/model_configs/33B.json -------------------------------------------------------------------------------- /configs/model_configs/65B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/model_configs/65B.json -------------------------------------------------------------------------------- /configs/model_configs/7B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/model_configs/7B.json -------------------------------------------------------------------------------- /configs/pretrain_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/pretrain_config.yaml -------------------------------------------------------------------------------- /configs/tokenizer_models/10w_vocab_wudao5_pile10.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/tokenizer_models/10w_vocab_wudao5_pile10.model -------------------------------------------------------------------------------- /configs/tokenizer_models/4w_cn_vocab_wudao15.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/tokenizer_models/4w_cn_vocab_wudao15.model -------------------------------------------------------------------------------- /configs/tokenizer_models/llama_tokenizer.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/tokenizer_models/llama_tokenizer.model -------------------------------------------------------------------------------- /configs/tokenizer_models/llama_tokenizer_extended.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/configs/tokenizer_models/llama_tokenizer_extended.model -------------------------------------------------------------------------------- /data/download_instruct.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/data/download_instruct.sh -------------------------------------------------------------------------------- /data/download_the_pile.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/data/download_the_pile.sh -------------------------------------------------------------------------------- /data/download_wudao.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/data/download_wudao.sh -------------------------------------------------------------------------------- /data/preprocess_instruction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/data/preprocess_instruction.py -------------------------------------------------------------------------------- /data/preprocess_the_pile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/data/preprocess_the_pile.py -------------------------------------------------------------------------------- /data/preprocess_wudao.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/data/preprocess_wudao.py -------------------------------------------------------------------------------- /dataset/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/dataset/dataset.py -------------------------------------------------------------------------------- /dataset/validation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/dataset/validation.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/requirements.txt -------------------------------------------------------------------------------- /solver/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/solver/trainer.py -------------------------------------------------------------------------------- /train_lm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/train_lm.py -------------------------------------------------------------------------------- /utils/convert_ckpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/convert_ckpt.py -------------------------------------------------------------------------------- /utils/merge_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/merge_tokenizer.py -------------------------------------------------------------------------------- /utils/speed_test/accelerate/ddp.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/accelerate/ddp.yaml -------------------------------------------------------------------------------- /utils/speed_test/accelerate/deepspeed_stage1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/accelerate/deepspeed_stage1.yaml -------------------------------------------------------------------------------- /utils/speed_test/accelerate/deepspeed_stage2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/accelerate/deepspeed_stage2.yaml -------------------------------------------------------------------------------- /utils/speed_test/accelerate/deepspeed_stage3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/accelerate/deepspeed_stage3.yaml -------------------------------------------------------------------------------- /utils/speed_test/accelerate/deepspeed_stage3_dynamo.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/accelerate/deepspeed_stage3_dynamo.yaml -------------------------------------------------------------------------------- /utils/speed_test/accelerate/deepspeed_stage3_offload.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/accelerate/deepspeed_stage3_offload.yaml -------------------------------------------------------------------------------- /utils/speed_test/accelerate/fsdp.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/accelerate/fsdp.yaml -------------------------------------------------------------------------------- /utils/speed_test/accelerate/megatron.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/accelerate/megatron.yaml -------------------------------------------------------------------------------- /utils/speed_test/accelerate/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/accelerate/run.py -------------------------------------------------------------------------------- /utils/speed_test/accelerate/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/accelerate/run.sh -------------------------------------------------------------------------------- /utils/speed_test/colossal-ai/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/colossal-ai/run.py -------------------------------------------------------------------------------- /utils/speed_test/colossal-ai/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/colossal-ai/run.sh -------------------------------------------------------------------------------- /utils/speed_test/colossal-ai/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/colossal-ai/utils.py -------------------------------------------------------------------------------- /utils/speed_test/lightning/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/speed_test/lightning/run.py -------------------------------------------------------------------------------- /utils/train_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidAI/Open-Llama/HEAD/utils/train_tokenizer.py --------------------------------------------------------------------------------