├── .github └── workflows │ └── checks.yml ├── .gitignore ├── LICENSE ├── README.md ├── configs └── pretraining │ ├── benchmark │ ├── llama2-70b_offload.yaml │ ├── llama2-7b.yaml │ ├── llama2-7b_offload.yaml │ └── mistral-7b.yaml │ ├── gpt2_demo.yaml │ ├── ja-stablelm-3b.yaml │ └── ja-stablelm-gamma-7b.yaml ├── convert_ckpt_to_hf.py ├── pretrain.py ├── pyproject.toml ├── src └── epochraft_hf_fsdp │ ├── __init__.py │ ├── config.py │ ├── fsdp.py │ ├── logging.py │ ├── lr_schedulers.py │ ├── pretraining_data.py │ └── trainer.py └── tox.ini /.github/workflows/checks.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/.github/workflows/checks.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/README.md -------------------------------------------------------------------------------- /configs/pretraining/benchmark/llama2-70b_offload.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/configs/pretraining/benchmark/llama2-70b_offload.yaml -------------------------------------------------------------------------------- /configs/pretraining/benchmark/llama2-7b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/configs/pretraining/benchmark/llama2-7b.yaml -------------------------------------------------------------------------------- /configs/pretraining/benchmark/llama2-7b_offload.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/configs/pretraining/benchmark/llama2-7b_offload.yaml -------------------------------------------------------------------------------- /configs/pretraining/benchmark/mistral-7b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/configs/pretraining/benchmark/mistral-7b.yaml -------------------------------------------------------------------------------- /configs/pretraining/gpt2_demo.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/configs/pretraining/gpt2_demo.yaml -------------------------------------------------------------------------------- /configs/pretraining/ja-stablelm-3b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/configs/pretraining/ja-stablelm-3b.yaml -------------------------------------------------------------------------------- /configs/pretraining/ja-stablelm-gamma-7b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/configs/pretraining/ja-stablelm-gamma-7b.yaml -------------------------------------------------------------------------------- /convert_ckpt_to_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/convert_ckpt_to_hf.py -------------------------------------------------------------------------------- /pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/pretrain.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/pyproject.toml -------------------------------------------------------------------------------- /src/epochraft_hf_fsdp/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/epochraft_hf_fsdp/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/src/epochraft_hf_fsdp/config.py -------------------------------------------------------------------------------- /src/epochraft_hf_fsdp/fsdp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/src/epochraft_hf_fsdp/fsdp.py -------------------------------------------------------------------------------- /src/epochraft_hf_fsdp/logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/src/epochraft_hf_fsdp/logging.py -------------------------------------------------------------------------------- /src/epochraft_hf_fsdp/lr_schedulers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/src/epochraft_hf_fsdp/lr_schedulers.py -------------------------------------------------------------------------------- /src/epochraft_hf_fsdp/pretraining_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/src/epochraft_hf_fsdp/pretraining_data.py -------------------------------------------------------------------------------- /src/epochraft_hf_fsdp/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/src/epochraft_hf_fsdp/trainer.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwiwi/epochraft-hf-fsdp/HEAD/tox.ini --------------------------------------------------------------------------------