├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── apps ├── __init__.py ├── aunet │ ├── README.md │ ├── __init__.py │ ├── assets │ │ └── AUNet.jpg │ ├── configs │ │ ├── 2B_1level.yaml │ │ └── 2B_2level.yaml │ ├── data │ │ ├── __init__.py │ │ ├── data.py │ │ └── regex_cutting.py │ ├── eval.py │ ├── generate.py │ ├── hierarchical.py │ ├── index_matmul.py │ └── train.py ├── fastRNN │ ├── component │ │ ├── compilable_scan.py │ │ └── rnn_common.py │ ├── eval.py │ ├── generate.py │ ├── hawk │ │ ├── configs │ │ │ ├── debug.yaml │ │ │ ├── hawk_1b.yaml │ │ │ └── hawk_7b.yaml │ │ ├── core_hawk.py │ │ └── hawk.py │ ├── minGRU │ │ ├── configs │ │ │ ├── debug.yaml │ │ │ └── minGRU_1b.yaml │ │ ├── core_gru.py │ │ └── mingru.py │ ├── minLSTM │ │ ├── configs │ │ │ ├── debug.yaml │ │ │ └── minLSTM_1b.yaml │ │ ├── core_lstm.py │ │ └── minlstm.py │ ├── requirements.txt │ └── train.py ├── main │ ├── __init__.py │ ├── configs │ │ ├── debug.yaml │ │ ├── eval.yaml │ │ ├── llama_1B.yaml │ │ └── llama_7B.yaml │ ├── eval.py │ ├── generate.py │ ├── train.py │ └── transformer.py ├── mamba │ ├── __init__.py │ ├── component │ │ ├── __init__.py │ │ ├── causal_conv1d_compilable.py │ │ └── ssm_compilable.py │ ├── configs │ │ ├── debug.yaml │ │ ├── eval.yaml │ │ ├── mamba_1b.yaml │ │ └── mamba_7b.yaml │ ├── core_mamba.py │ ├── eval.py │ ├── generate.py │ ├── mamba.py │ ├── requirements.txt │ └── train.py ├── mtp │ ├── configs │ │ └── debug.yaml │ ├── eval.py │ ├── train.py │ └── transformer.py └── plots │ ├── __init__.py │ ├── analysis.py │ └── probe_animation.py ├── dataloader.png ├── lingua ├── __init__.py ├── args.py ├── checkpoint.py ├── data.py ├── distributed.py ├── float8.py ├── logger.py ├── metrics.py ├── optim.py ├── probe.py ├── profiling.py ├── stool.py ├── tokenizer.py └── transformer.py ├── lingua_overview.svg ├── requirements.txt └── setup ├── create_env.sh ├── download_prepare_hf_data.py └── download_tokenizer.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .vscode 3 | *.ipynb 4 | slurm-*.out 5 | wandb -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/README.md -------------------------------------------------------------------------------- /apps/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /apps/aunet/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/aunet/README.md -------------------------------------------------------------------------------- /apps/aunet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /apps/aunet/assets/AUNet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/aunet/assets/AUNet.jpg -------------------------------------------------------------------------------- /apps/aunet/configs/2B_1level.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/aunet/configs/2B_1level.yaml -------------------------------------------------------------------------------- /apps/aunet/configs/2B_2level.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/aunet/configs/2B_2level.yaml -------------------------------------------------------------------------------- /apps/aunet/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /apps/aunet/data/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/aunet/data/data.py -------------------------------------------------------------------------------- /apps/aunet/data/regex_cutting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/aunet/data/regex_cutting.py -------------------------------------------------------------------------------- /apps/aunet/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/aunet/eval.py -------------------------------------------------------------------------------- /apps/aunet/generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/aunet/generate.py -------------------------------------------------------------------------------- /apps/aunet/hierarchical.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/aunet/hierarchical.py -------------------------------------------------------------------------------- /apps/aunet/index_matmul.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/aunet/index_matmul.py -------------------------------------------------------------------------------- /apps/aunet/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/aunet/train.py -------------------------------------------------------------------------------- /apps/fastRNN/component/compilable_scan.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/component/compilable_scan.py -------------------------------------------------------------------------------- /apps/fastRNN/component/rnn_common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/component/rnn_common.py -------------------------------------------------------------------------------- /apps/fastRNN/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/eval.py -------------------------------------------------------------------------------- /apps/fastRNN/generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/generate.py -------------------------------------------------------------------------------- /apps/fastRNN/hawk/configs/debug.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/hawk/configs/debug.yaml -------------------------------------------------------------------------------- /apps/fastRNN/hawk/configs/hawk_1b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/hawk/configs/hawk_1b.yaml -------------------------------------------------------------------------------- /apps/fastRNN/hawk/configs/hawk_7b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/hawk/configs/hawk_7b.yaml -------------------------------------------------------------------------------- /apps/fastRNN/hawk/core_hawk.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/hawk/core_hawk.py -------------------------------------------------------------------------------- /apps/fastRNN/hawk/hawk.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/hawk/hawk.py -------------------------------------------------------------------------------- /apps/fastRNN/minGRU/configs/debug.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/minGRU/configs/debug.yaml -------------------------------------------------------------------------------- /apps/fastRNN/minGRU/configs/minGRU_1b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/minGRU/configs/minGRU_1b.yaml -------------------------------------------------------------------------------- /apps/fastRNN/minGRU/core_gru.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/minGRU/core_gru.py -------------------------------------------------------------------------------- /apps/fastRNN/minGRU/mingru.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/minGRU/mingru.py -------------------------------------------------------------------------------- /apps/fastRNN/minLSTM/configs/debug.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/minLSTM/configs/debug.yaml -------------------------------------------------------------------------------- /apps/fastRNN/minLSTM/configs/minLSTM_1b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/minLSTM/configs/minLSTM_1b.yaml -------------------------------------------------------------------------------- /apps/fastRNN/minLSTM/core_lstm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/minLSTM/core_lstm.py -------------------------------------------------------------------------------- /apps/fastRNN/minLSTM/minlstm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/minLSTM/minlstm.py -------------------------------------------------------------------------------- /apps/fastRNN/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/requirements.txt -------------------------------------------------------------------------------- /apps/fastRNN/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/fastRNN/train.py -------------------------------------------------------------------------------- /apps/main/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /apps/main/configs/debug.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/main/configs/debug.yaml -------------------------------------------------------------------------------- /apps/main/configs/eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/main/configs/eval.yaml -------------------------------------------------------------------------------- /apps/main/configs/llama_1B.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/main/configs/llama_1B.yaml -------------------------------------------------------------------------------- /apps/main/configs/llama_7B.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/main/configs/llama_7B.yaml -------------------------------------------------------------------------------- /apps/main/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/main/eval.py -------------------------------------------------------------------------------- /apps/main/generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/main/generate.py -------------------------------------------------------------------------------- /apps/main/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/main/train.py -------------------------------------------------------------------------------- /apps/main/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/main/transformer.py -------------------------------------------------------------------------------- /apps/mamba/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /apps/mamba/component/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /apps/mamba/component/causal_conv1d_compilable.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/component/causal_conv1d_compilable.py -------------------------------------------------------------------------------- /apps/mamba/component/ssm_compilable.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/component/ssm_compilable.py -------------------------------------------------------------------------------- /apps/mamba/configs/debug.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/configs/debug.yaml -------------------------------------------------------------------------------- /apps/mamba/configs/eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/configs/eval.yaml -------------------------------------------------------------------------------- /apps/mamba/configs/mamba_1b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/configs/mamba_1b.yaml -------------------------------------------------------------------------------- /apps/mamba/configs/mamba_7b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/configs/mamba_7b.yaml -------------------------------------------------------------------------------- /apps/mamba/core_mamba.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/core_mamba.py -------------------------------------------------------------------------------- /apps/mamba/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/eval.py -------------------------------------------------------------------------------- /apps/mamba/generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/generate.py -------------------------------------------------------------------------------- /apps/mamba/mamba.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/mamba.py -------------------------------------------------------------------------------- /apps/mamba/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/requirements.txt -------------------------------------------------------------------------------- /apps/mamba/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mamba/train.py -------------------------------------------------------------------------------- /apps/mtp/configs/debug.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mtp/configs/debug.yaml -------------------------------------------------------------------------------- /apps/mtp/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mtp/eval.py -------------------------------------------------------------------------------- /apps/mtp/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mtp/train.py -------------------------------------------------------------------------------- /apps/mtp/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/mtp/transformer.py -------------------------------------------------------------------------------- /apps/plots/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /apps/plots/analysis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/plots/analysis.py -------------------------------------------------------------------------------- /apps/plots/probe_animation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/apps/plots/probe_animation.py -------------------------------------------------------------------------------- /dataloader.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/dataloader.png -------------------------------------------------------------------------------- /lingua/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lingua/args.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/args.py -------------------------------------------------------------------------------- /lingua/checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/checkpoint.py -------------------------------------------------------------------------------- /lingua/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/data.py -------------------------------------------------------------------------------- /lingua/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/distributed.py -------------------------------------------------------------------------------- /lingua/float8.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/float8.py -------------------------------------------------------------------------------- /lingua/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/logger.py -------------------------------------------------------------------------------- /lingua/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/metrics.py -------------------------------------------------------------------------------- /lingua/optim.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/optim.py -------------------------------------------------------------------------------- /lingua/probe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/probe.py -------------------------------------------------------------------------------- /lingua/profiling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/profiling.py -------------------------------------------------------------------------------- /lingua/stool.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/stool.py -------------------------------------------------------------------------------- /lingua/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/tokenizer.py -------------------------------------------------------------------------------- /lingua/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua/transformer.py -------------------------------------------------------------------------------- /lingua_overview.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/lingua_overview.svg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/requirements.txt -------------------------------------------------------------------------------- /setup/create_env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/setup/create_env.sh -------------------------------------------------------------------------------- /setup/download_prepare_hf_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/setup/download_prepare_hf_data.py -------------------------------------------------------------------------------- /setup/download_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/lingua/HEAD/setup/download_tokenizer.py --------------------------------------------------------------------------------