├── .gitignore ├── LICENSE ├── README.md ├── assets ├── babyllama-dalle3.png └── wandb-Llama-gptj-gpt2.png ├── babylm_dataset.py ├── cleaning_and_tokenization.ipynb ├── config ├── gpt-705M.yaml ├── gpt-97M-strict.yaml ├── gpt-97M.yaml ├── gpt-j-97M-strict.yaml ├── gpt-j-97M.yaml ├── llama-16M.yaml ├── llama-360M.yaml ├── llama-95M-strict.yaml ├── llama-95M.yaml ├── llama-97M-strict.yaml ├── llama-97M.yaml └── llama-bigger-strict.yaml ├── distill-ensemble-pretraining-baby-llama.py ├── mrclean.py └── train.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/README.md -------------------------------------------------------------------------------- /assets/babyllama-dalle3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/assets/babyllama-dalle3.png -------------------------------------------------------------------------------- /assets/wandb-Llama-gptj-gpt2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/assets/wandb-Llama-gptj-gpt2.png -------------------------------------------------------------------------------- /babylm_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/babylm_dataset.py -------------------------------------------------------------------------------- /cleaning_and_tokenization.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/cleaning_and_tokenization.ipynb -------------------------------------------------------------------------------- /config/gpt-705M.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/gpt-705M.yaml -------------------------------------------------------------------------------- /config/gpt-97M-strict.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/gpt-97M-strict.yaml -------------------------------------------------------------------------------- /config/gpt-97M.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/gpt-97M.yaml -------------------------------------------------------------------------------- /config/gpt-j-97M-strict.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/gpt-j-97M-strict.yaml -------------------------------------------------------------------------------- /config/gpt-j-97M.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/gpt-j-97M.yaml -------------------------------------------------------------------------------- /config/llama-16M.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/llama-16M.yaml -------------------------------------------------------------------------------- /config/llama-360M.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/llama-360M.yaml -------------------------------------------------------------------------------- /config/llama-95M-strict.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/llama-95M-strict.yaml -------------------------------------------------------------------------------- /config/llama-95M.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/llama-95M.yaml -------------------------------------------------------------------------------- /config/llama-97M-strict.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/llama-97M-strict.yaml -------------------------------------------------------------------------------- /config/llama-97M.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/llama-97M.yaml -------------------------------------------------------------------------------- /config/llama-bigger-strict.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/config/llama-bigger-strict.yaml -------------------------------------------------------------------------------- /distill-ensemble-pretraining-baby-llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/distill-ensemble-pretraining-baby-llama.py -------------------------------------------------------------------------------- /mrclean.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/mrclean.py -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timinar/BabyLlama/HEAD/train.py --------------------------------------------------------------------------------