├── .gitignore ├── LICENSE ├── README.md ├── original └── nanoGPT │ ├── LICENSE │ ├── README.md │ ├── assets │ ├── gpt2_124M_loss.png │ └── nanogpt.jpg │ ├── bench.py │ ├── config │ ├── eval_gpt2.py │ ├── eval_gpt2_large.py │ ├── eval_gpt2_medium.py │ ├── eval_gpt2_xl.py │ ├── finetune_shakespeare.py │ ├── train_gpt2.py │ └── train_shakespeare_char.py │ ├── configurator.py │ ├── data │ ├── openwebtext │ │ ├── prepare.py │ │ └── readme.md │ ├── shakespeare │ │ ├── prepare.py │ │ └── readme.md │ └── shakespeare_char │ │ ├── prepare.py │ │ └── readme.md │ ├── finetune_glue │ ├── load_hf_model.py │ ├── model.py │ ├── run.sh │ └── run_glue_no_trainer.py │ ├── finetune_squad │ ├── load_hf_model.py │ ├── model.py │ ├── run.sh │ ├── run_qa_no_trainer.py │ └── utils_qa.py │ ├── model.py │ ├── sample.py │ ├── scaling_laws.ipynb │ ├── train.py │ └── transformer_sizing.ipynb ├── v1 └── nanoGPT │ ├── LICENSE │ ├── README.md │ ├── assets │ ├── gpt2_124M_loss.png │ └── nanogpt.jpg │ ├── bench.py │ ├── config │ ├── eval_gpt2.py │ ├── eval_gpt2_large.py │ ├── eval_gpt2_medium.py │ ├── eval_gpt2_xl.py │ ├── finetune_shakespeare.py │ ├── train_gpt2.py │ └── train_shakespeare_char.py │ ├── configurator.py │ ├── data │ ├── openwebtext │ │ ├── prepare.py │ │ └── readme.md │ ├── shakespeare │ │ ├── prepare.py │ │ └── readme.md │ └── shakespeare_char │ │ ├── prepare.py │ │ └── readme.md │ ├── finetune_glue │ ├── load_hf_model.py │ ├── model.py │ ├── run.sh │ ├── run_glue_no_trainer.py │ └── sparse_ops.py │ ├── finetune_squad │ ├── load_hf_model.py │ ├── model.py │ ├── run.sh │ ├── run_qa_no_trainer.py │ ├── sparse_ops.py │ └── utils_qa.py │ ├── model.py │ ├── sample.py │ ├── scaling_laws.ipynb │ ├── sparse_ops.py │ ├── train.py │ └── transformer_sizing.ipynb └── v2 └── nanoGPT ├── LICENSE ├── README.md ├── assets ├── gpt2_124M_loss.png └── nanogpt.jpg ├── bench.py ├── config ├── eval_gpt2.py ├── eval_gpt2_large.py ├── eval_gpt2_medium.py ├── eval_gpt2_xl.py ├── finetune_shakespeare.py ├── train_gpt2.py └── train_shakespeare_char.py ├── configurator.py ├── data ├── openwebtext │ ├── prepare.py │ └── readme.md ├── shakespeare │ ├── prepare.py │ └── readme.md └── shakespeare_char │ ├── prepare.py │ └── readme.md ├── finetune_glue ├── load_hf_model.py ├── model.py ├── run.sh ├── run_glue_no_trainer.py └── sparse_ops.py ├── finetune_squad ├── load_hf_model.py ├── model.py ├── run.sh ├── run_qa_no_trainer.py ├── sparse_ops.py └── utils_qa.py ├── model.py ├── sample.py ├── scaling_laws.ipynb ├── sparse_ops.py ├── test.py ├── train.py └── transformer_sizing.ipynb /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/README.md -------------------------------------------------------------------------------- /original/nanoGPT/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/LICENSE -------------------------------------------------------------------------------- /original/nanoGPT/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/README.md -------------------------------------------------------------------------------- /original/nanoGPT/assets/gpt2_124M_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/assets/gpt2_124M_loss.png -------------------------------------------------------------------------------- /original/nanoGPT/assets/nanogpt.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/assets/nanogpt.jpg -------------------------------------------------------------------------------- /original/nanoGPT/bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/bench.py -------------------------------------------------------------------------------- /original/nanoGPT/config/eval_gpt2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/config/eval_gpt2.py -------------------------------------------------------------------------------- /original/nanoGPT/config/eval_gpt2_large.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/config/eval_gpt2_large.py -------------------------------------------------------------------------------- /original/nanoGPT/config/eval_gpt2_medium.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/config/eval_gpt2_medium.py -------------------------------------------------------------------------------- /original/nanoGPT/config/eval_gpt2_xl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/config/eval_gpt2_xl.py -------------------------------------------------------------------------------- /original/nanoGPT/config/finetune_shakespeare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/config/finetune_shakespeare.py -------------------------------------------------------------------------------- /original/nanoGPT/config/train_gpt2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/config/train_gpt2.py -------------------------------------------------------------------------------- /original/nanoGPT/config/train_shakespeare_char.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/config/train_shakespeare_char.py -------------------------------------------------------------------------------- /original/nanoGPT/configurator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/configurator.py -------------------------------------------------------------------------------- /original/nanoGPT/data/openwebtext/prepare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/data/openwebtext/prepare.py -------------------------------------------------------------------------------- /original/nanoGPT/data/openwebtext/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/data/openwebtext/readme.md -------------------------------------------------------------------------------- /original/nanoGPT/data/shakespeare/prepare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/data/shakespeare/prepare.py -------------------------------------------------------------------------------- /original/nanoGPT/data/shakespeare/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/data/shakespeare/readme.md -------------------------------------------------------------------------------- /original/nanoGPT/data/shakespeare_char/prepare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/data/shakespeare_char/prepare.py -------------------------------------------------------------------------------- /original/nanoGPT/data/shakespeare_char/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/data/shakespeare_char/readme.md -------------------------------------------------------------------------------- /original/nanoGPT/finetune_glue/load_hf_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/finetune_glue/load_hf_model.py -------------------------------------------------------------------------------- /original/nanoGPT/finetune_glue/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/finetune_glue/model.py -------------------------------------------------------------------------------- /original/nanoGPT/finetune_glue/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/finetune_glue/run.sh -------------------------------------------------------------------------------- /original/nanoGPT/finetune_glue/run_glue_no_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/finetune_glue/run_glue_no_trainer.py -------------------------------------------------------------------------------- /original/nanoGPT/finetune_squad/load_hf_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/finetune_squad/load_hf_model.py -------------------------------------------------------------------------------- /original/nanoGPT/finetune_squad/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/finetune_squad/model.py -------------------------------------------------------------------------------- /original/nanoGPT/finetune_squad/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/finetune_squad/run.sh -------------------------------------------------------------------------------- /original/nanoGPT/finetune_squad/run_qa_no_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/finetune_squad/run_qa_no_trainer.py -------------------------------------------------------------------------------- /original/nanoGPT/finetune_squad/utils_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/finetune_squad/utils_qa.py -------------------------------------------------------------------------------- /original/nanoGPT/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/model.py -------------------------------------------------------------------------------- /original/nanoGPT/sample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/sample.py -------------------------------------------------------------------------------- /original/nanoGPT/scaling_laws.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/scaling_laws.ipynb -------------------------------------------------------------------------------- /original/nanoGPT/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/train.py -------------------------------------------------------------------------------- /original/nanoGPT/transformer_sizing.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/original/nanoGPT/transformer_sizing.ipynb -------------------------------------------------------------------------------- /v1/nanoGPT/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/LICENSE -------------------------------------------------------------------------------- /v1/nanoGPT/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/README.md -------------------------------------------------------------------------------- /v1/nanoGPT/assets/gpt2_124M_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/assets/gpt2_124M_loss.png -------------------------------------------------------------------------------- /v1/nanoGPT/assets/nanogpt.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/assets/nanogpt.jpg -------------------------------------------------------------------------------- /v1/nanoGPT/bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/bench.py -------------------------------------------------------------------------------- /v1/nanoGPT/config/eval_gpt2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/config/eval_gpt2.py -------------------------------------------------------------------------------- /v1/nanoGPT/config/eval_gpt2_large.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/config/eval_gpt2_large.py -------------------------------------------------------------------------------- /v1/nanoGPT/config/eval_gpt2_medium.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/config/eval_gpt2_medium.py -------------------------------------------------------------------------------- /v1/nanoGPT/config/eval_gpt2_xl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/config/eval_gpt2_xl.py -------------------------------------------------------------------------------- /v1/nanoGPT/config/finetune_shakespeare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/config/finetune_shakespeare.py -------------------------------------------------------------------------------- /v1/nanoGPT/config/train_gpt2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/config/train_gpt2.py -------------------------------------------------------------------------------- /v1/nanoGPT/config/train_shakespeare_char.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/config/train_shakespeare_char.py -------------------------------------------------------------------------------- /v1/nanoGPT/configurator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/configurator.py -------------------------------------------------------------------------------- /v1/nanoGPT/data/openwebtext/prepare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/data/openwebtext/prepare.py -------------------------------------------------------------------------------- /v1/nanoGPT/data/openwebtext/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/data/openwebtext/readme.md -------------------------------------------------------------------------------- /v1/nanoGPT/data/shakespeare/prepare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/data/shakespeare/prepare.py -------------------------------------------------------------------------------- /v1/nanoGPT/data/shakespeare/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/data/shakespeare/readme.md -------------------------------------------------------------------------------- /v1/nanoGPT/data/shakespeare_char/prepare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/data/shakespeare_char/prepare.py -------------------------------------------------------------------------------- /v1/nanoGPT/data/shakespeare_char/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/data/shakespeare_char/readme.md -------------------------------------------------------------------------------- /v1/nanoGPT/finetune_glue/load_hf_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/finetune_glue/load_hf_model.py -------------------------------------------------------------------------------- /v1/nanoGPT/finetune_glue/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/finetune_glue/model.py -------------------------------------------------------------------------------- /v1/nanoGPT/finetune_glue/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/finetune_glue/run.sh -------------------------------------------------------------------------------- /v1/nanoGPT/finetune_glue/run_glue_no_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/finetune_glue/run_glue_no_trainer.py -------------------------------------------------------------------------------- /v1/nanoGPT/finetune_glue/sparse_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/finetune_glue/sparse_ops.py -------------------------------------------------------------------------------- /v1/nanoGPT/finetune_squad/load_hf_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/finetune_squad/load_hf_model.py -------------------------------------------------------------------------------- /v1/nanoGPT/finetune_squad/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/finetune_squad/model.py -------------------------------------------------------------------------------- /v1/nanoGPT/finetune_squad/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/finetune_squad/run.sh -------------------------------------------------------------------------------- /v1/nanoGPT/finetune_squad/run_qa_no_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/finetune_squad/run_qa_no_trainer.py -------------------------------------------------------------------------------- /v1/nanoGPT/finetune_squad/sparse_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/finetune_squad/sparse_ops.py -------------------------------------------------------------------------------- /v1/nanoGPT/finetune_squad/utils_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/finetune_squad/utils_qa.py -------------------------------------------------------------------------------- /v1/nanoGPT/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/model.py -------------------------------------------------------------------------------- /v1/nanoGPT/sample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/sample.py -------------------------------------------------------------------------------- /v1/nanoGPT/scaling_laws.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/scaling_laws.ipynb -------------------------------------------------------------------------------- /v1/nanoGPT/sparse_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/sparse_ops.py -------------------------------------------------------------------------------- /v1/nanoGPT/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/train.py -------------------------------------------------------------------------------- /v1/nanoGPT/transformer_sizing.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v1/nanoGPT/transformer_sizing.ipynb -------------------------------------------------------------------------------- /v2/nanoGPT/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/LICENSE -------------------------------------------------------------------------------- /v2/nanoGPT/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/README.md -------------------------------------------------------------------------------- /v2/nanoGPT/assets/gpt2_124M_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/assets/gpt2_124M_loss.png -------------------------------------------------------------------------------- /v2/nanoGPT/assets/nanogpt.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/assets/nanogpt.jpg -------------------------------------------------------------------------------- /v2/nanoGPT/bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/bench.py -------------------------------------------------------------------------------- /v2/nanoGPT/config/eval_gpt2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/config/eval_gpt2.py -------------------------------------------------------------------------------- /v2/nanoGPT/config/eval_gpt2_large.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/config/eval_gpt2_large.py -------------------------------------------------------------------------------- /v2/nanoGPT/config/eval_gpt2_medium.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/config/eval_gpt2_medium.py -------------------------------------------------------------------------------- /v2/nanoGPT/config/eval_gpt2_xl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/config/eval_gpt2_xl.py -------------------------------------------------------------------------------- /v2/nanoGPT/config/finetune_shakespeare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/config/finetune_shakespeare.py -------------------------------------------------------------------------------- /v2/nanoGPT/config/train_gpt2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/config/train_gpt2.py -------------------------------------------------------------------------------- /v2/nanoGPT/config/train_shakespeare_char.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/config/train_shakespeare_char.py -------------------------------------------------------------------------------- /v2/nanoGPT/configurator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/configurator.py -------------------------------------------------------------------------------- /v2/nanoGPT/data/openwebtext/prepare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/data/openwebtext/prepare.py -------------------------------------------------------------------------------- /v2/nanoGPT/data/openwebtext/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/data/openwebtext/readme.md -------------------------------------------------------------------------------- /v2/nanoGPT/data/shakespeare/prepare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/data/shakespeare/prepare.py -------------------------------------------------------------------------------- /v2/nanoGPT/data/shakespeare/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/data/shakespeare/readme.md -------------------------------------------------------------------------------- /v2/nanoGPT/data/shakespeare_char/prepare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/data/shakespeare_char/prepare.py -------------------------------------------------------------------------------- /v2/nanoGPT/data/shakespeare_char/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/data/shakespeare_char/readme.md -------------------------------------------------------------------------------- /v2/nanoGPT/finetune_glue/load_hf_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/finetune_glue/load_hf_model.py -------------------------------------------------------------------------------- /v2/nanoGPT/finetune_glue/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/finetune_glue/model.py -------------------------------------------------------------------------------- /v2/nanoGPT/finetune_glue/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/finetune_glue/run.sh -------------------------------------------------------------------------------- /v2/nanoGPT/finetune_glue/run_glue_no_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/finetune_glue/run_glue_no_trainer.py -------------------------------------------------------------------------------- /v2/nanoGPT/finetune_glue/sparse_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/finetune_glue/sparse_ops.py -------------------------------------------------------------------------------- /v2/nanoGPT/finetune_squad/load_hf_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/finetune_squad/load_hf_model.py -------------------------------------------------------------------------------- /v2/nanoGPT/finetune_squad/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/finetune_squad/model.py -------------------------------------------------------------------------------- /v2/nanoGPT/finetune_squad/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/finetune_squad/run.sh -------------------------------------------------------------------------------- /v2/nanoGPT/finetune_squad/run_qa_no_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/finetune_squad/run_qa_no_trainer.py -------------------------------------------------------------------------------- /v2/nanoGPT/finetune_squad/sparse_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/finetune_squad/sparse_ops.py -------------------------------------------------------------------------------- /v2/nanoGPT/finetune_squad/utils_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/finetune_squad/utils_qa.py -------------------------------------------------------------------------------- /v2/nanoGPT/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/model.py -------------------------------------------------------------------------------- /v2/nanoGPT/sample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/sample.py -------------------------------------------------------------------------------- /v2/nanoGPT/scaling_laws.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/scaling_laws.ipynb -------------------------------------------------------------------------------- /v2/nanoGPT/sparse_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/sparse_ops.py -------------------------------------------------------------------------------- /v2/nanoGPT/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/test.py -------------------------------------------------------------------------------- /v2/nanoGPT/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/train.py -------------------------------------------------------------------------------- /v2/nanoGPT/transformer_sizing.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/2by4-pretrain-acc-examples/HEAD/v2/nanoGPT/transformer_sizing.ipynb --------------------------------------------------------------------------------