├── .gitignore ├── LICENSE ├── README.md ├── config ├── dpo │ └── MiniLLM-0.2B-DPO │ │ ├── bert4torch_config.json │ │ ├── config.json │ │ ├── dpo_args.yaml │ │ └── generation_config.json ├── pretrain │ ├── MiniLLM-0.2B-Base │ │ ├── bert4torch_config.json │ │ ├── config.json │ │ ├── ds_config.json │ │ └── pretrain_args.yaml │ ├── MiniLLM-0.2B-NoWudao-Base │ │ ├── bert4torch_config.json │ │ ├── config.json │ │ └── pretrain_args.yaml │ └── MiniLLM-1.1B-Base │ │ ├── bert4torch_config.json │ │ ├── config.json │ │ └── pretrain_args.yaml └── sft │ ├── MiniLLM-0.2B-SFT-Alpaca │ ├── bert4torch_config.json │ ├── config.json │ ├── generation_config.json │ └── sft_args.yaml │ ├── MiniLLM-0.2B-SFT │ ├── bert4torch_config.json │ ├── config.json │ ├── generation_config.json │ └── sft_args.yaml │ └── MiniLLM-1.1B-SFT │ ├── bert4torch_config.json │ ├── config.json │ └── sft_args.yaml ├── docs ├── convert.py └── pics │ ├── gradio_demo.gif │ ├── minillm.png │ ├── tensorboard.png │ ├── tensorboard_sft.png │ ├── tensorboard_sft1.png │ ├── wechat.jpg │ └── wechat_group.jpg ├── dpo ├── data_process.py ├── dpo.py └── infer.py ├── pretrain ├── data_process.py ├── infer.py ├── infer_transformers.py └── pretrain.py ├── requirement.txt ├── sft ├── data_process.py ├── infer.py ├── infer_transformers.py ├── sft.py └── webui.py └── tokenizer ├── tokenization_chatglm.py ├── tokenizer.model └── tokenizer_config.json /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/README.md -------------------------------------------------------------------------------- /config/dpo/MiniLLM-0.2B-DPO/bert4torch_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/dpo/MiniLLM-0.2B-DPO/bert4torch_config.json -------------------------------------------------------------------------------- /config/dpo/MiniLLM-0.2B-DPO/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/dpo/MiniLLM-0.2B-DPO/config.json -------------------------------------------------------------------------------- /config/dpo/MiniLLM-0.2B-DPO/dpo_args.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/dpo/MiniLLM-0.2B-DPO/dpo_args.yaml -------------------------------------------------------------------------------- /config/dpo/MiniLLM-0.2B-DPO/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/dpo/MiniLLM-0.2B-DPO/generation_config.json -------------------------------------------------------------------------------- /config/pretrain/MiniLLM-0.2B-Base/bert4torch_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/pretrain/MiniLLM-0.2B-Base/bert4torch_config.json -------------------------------------------------------------------------------- /config/pretrain/MiniLLM-0.2B-Base/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/pretrain/MiniLLM-0.2B-Base/config.json -------------------------------------------------------------------------------- /config/pretrain/MiniLLM-0.2B-Base/ds_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/pretrain/MiniLLM-0.2B-Base/ds_config.json -------------------------------------------------------------------------------- /config/pretrain/MiniLLM-0.2B-Base/pretrain_args.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/pretrain/MiniLLM-0.2B-Base/pretrain_args.yaml -------------------------------------------------------------------------------- /config/pretrain/MiniLLM-0.2B-NoWudao-Base/bert4torch_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/pretrain/MiniLLM-0.2B-NoWudao-Base/bert4torch_config.json -------------------------------------------------------------------------------- /config/pretrain/MiniLLM-0.2B-NoWudao-Base/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/pretrain/MiniLLM-0.2B-NoWudao-Base/config.json -------------------------------------------------------------------------------- /config/pretrain/MiniLLM-0.2B-NoWudao-Base/pretrain_args.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/pretrain/MiniLLM-0.2B-NoWudao-Base/pretrain_args.yaml -------------------------------------------------------------------------------- /config/pretrain/MiniLLM-1.1B-Base/bert4torch_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/pretrain/MiniLLM-1.1B-Base/bert4torch_config.json -------------------------------------------------------------------------------- /config/pretrain/MiniLLM-1.1B-Base/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/pretrain/MiniLLM-1.1B-Base/config.json -------------------------------------------------------------------------------- /config/pretrain/MiniLLM-1.1B-Base/pretrain_args.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/pretrain/MiniLLM-1.1B-Base/pretrain_args.yaml -------------------------------------------------------------------------------- /config/sft/MiniLLM-0.2B-SFT-Alpaca/bert4torch_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/sft/MiniLLM-0.2B-SFT-Alpaca/bert4torch_config.json -------------------------------------------------------------------------------- /config/sft/MiniLLM-0.2B-SFT-Alpaca/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/sft/MiniLLM-0.2B-SFT-Alpaca/config.json -------------------------------------------------------------------------------- /config/sft/MiniLLM-0.2B-SFT-Alpaca/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/sft/MiniLLM-0.2B-SFT-Alpaca/generation_config.json -------------------------------------------------------------------------------- /config/sft/MiniLLM-0.2B-SFT-Alpaca/sft_args.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/sft/MiniLLM-0.2B-SFT-Alpaca/sft_args.yaml -------------------------------------------------------------------------------- /config/sft/MiniLLM-0.2B-SFT/bert4torch_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/sft/MiniLLM-0.2B-SFT/bert4torch_config.json -------------------------------------------------------------------------------- /config/sft/MiniLLM-0.2B-SFT/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/sft/MiniLLM-0.2B-SFT/config.json -------------------------------------------------------------------------------- /config/sft/MiniLLM-0.2B-SFT/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/sft/MiniLLM-0.2B-SFT/generation_config.json -------------------------------------------------------------------------------- /config/sft/MiniLLM-0.2B-SFT/sft_args.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/sft/MiniLLM-0.2B-SFT/sft_args.yaml -------------------------------------------------------------------------------- /config/sft/MiniLLM-1.1B-SFT/bert4torch_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/sft/MiniLLM-1.1B-SFT/bert4torch_config.json -------------------------------------------------------------------------------- /config/sft/MiniLLM-1.1B-SFT/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/sft/MiniLLM-1.1B-SFT/config.json -------------------------------------------------------------------------------- /config/sft/MiniLLM-1.1B-SFT/sft_args.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/config/sft/MiniLLM-1.1B-SFT/sft_args.yaml -------------------------------------------------------------------------------- /docs/convert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/docs/convert.py -------------------------------------------------------------------------------- /docs/pics/gradio_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/docs/pics/gradio_demo.gif -------------------------------------------------------------------------------- /docs/pics/minillm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/docs/pics/minillm.png -------------------------------------------------------------------------------- /docs/pics/tensorboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/docs/pics/tensorboard.png -------------------------------------------------------------------------------- /docs/pics/tensorboard_sft.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/docs/pics/tensorboard_sft.png -------------------------------------------------------------------------------- /docs/pics/tensorboard_sft1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/docs/pics/tensorboard_sft1.png -------------------------------------------------------------------------------- /docs/pics/wechat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/docs/pics/wechat.jpg -------------------------------------------------------------------------------- /docs/pics/wechat_group.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/docs/pics/wechat_group.jpg -------------------------------------------------------------------------------- /dpo/data_process.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/dpo/data_process.py -------------------------------------------------------------------------------- /dpo/dpo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/dpo/dpo.py -------------------------------------------------------------------------------- /dpo/infer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/dpo/infer.py -------------------------------------------------------------------------------- /pretrain/data_process.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/pretrain/data_process.py -------------------------------------------------------------------------------- /pretrain/infer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/pretrain/infer.py -------------------------------------------------------------------------------- /pretrain/infer_transformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/pretrain/infer_transformers.py -------------------------------------------------------------------------------- /pretrain/pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/pretrain/pretrain.py -------------------------------------------------------------------------------- /requirement.txt: -------------------------------------------------------------------------------- 1 | bert4torch==0.5.5 2 | transformers==4.44.0 -------------------------------------------------------------------------------- /sft/data_process.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/sft/data_process.py -------------------------------------------------------------------------------- /sft/infer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/sft/infer.py -------------------------------------------------------------------------------- /sft/infer_transformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/sft/infer_transformers.py -------------------------------------------------------------------------------- /sft/sft.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/sft/sft.py -------------------------------------------------------------------------------- /sft/webui.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/sft/webui.py -------------------------------------------------------------------------------- /tokenizer/tokenization_chatglm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/tokenizer/tokenization_chatglm.py -------------------------------------------------------------------------------- /tokenizer/tokenizer.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/tokenizer/tokenizer.model -------------------------------------------------------------------------------- /tokenizer/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tongjilibo/build_MiniLLM_from_scratch/HEAD/tokenizer/tokenizer_config.json --------------------------------------------------------------------------------