├── LICENSE ├── README.md ├── benchmarks ├── communication │ ├── README.md │ ├── __init__.py │ ├── all_reduce.py │ ├── broadcast.py │ ├── constants.py │ ├── pt2pt.py │ ├── run_all.py │ └── utils.py └── computation │ ├── README.md │ ├── benchmark_flash_attention.py │ ├── benchmark_mamba.py │ ├── benchmark_mamba2.py │ └── utils.py ├── calc ├── README.md ├── calc_mamba_flops.py ├── calc_mamba_params.py └── data │ ├── convert_into_jsonl_partitions.py │ └── tokenize_and_count.py └── imgs ├── annealing-example.png ├── mamba-moe.png ├── mamba.png ├── transformer-moe.png ├── transformer.png ├── zamba-7b.png ├── zamba2-1p2b.png ├── zamba2-2p7b.png └── zcookbook.jpg /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/README.md -------------------------------------------------------------------------------- /benchmarks/communication/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/communication/README.md -------------------------------------------------------------------------------- /benchmarks/communication/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchmarks/communication/all_reduce.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/communication/all_reduce.py -------------------------------------------------------------------------------- /benchmarks/communication/broadcast.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/communication/broadcast.py -------------------------------------------------------------------------------- /benchmarks/communication/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/communication/constants.py -------------------------------------------------------------------------------- /benchmarks/communication/pt2pt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/communication/pt2pt.py -------------------------------------------------------------------------------- /benchmarks/communication/run_all.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/communication/run_all.py -------------------------------------------------------------------------------- /benchmarks/communication/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/communication/utils.py -------------------------------------------------------------------------------- /benchmarks/computation/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/computation/README.md -------------------------------------------------------------------------------- /benchmarks/computation/benchmark_flash_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/computation/benchmark_flash_attention.py -------------------------------------------------------------------------------- /benchmarks/computation/benchmark_mamba.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/computation/benchmark_mamba.py -------------------------------------------------------------------------------- /benchmarks/computation/benchmark_mamba2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/computation/benchmark_mamba2.py -------------------------------------------------------------------------------- /benchmarks/computation/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/benchmarks/computation/utils.py -------------------------------------------------------------------------------- /calc/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/calc/README.md -------------------------------------------------------------------------------- /calc/calc_mamba_flops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/calc/calc_mamba_flops.py -------------------------------------------------------------------------------- /calc/calc_mamba_params.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/calc/calc_mamba_params.py -------------------------------------------------------------------------------- /calc/data/convert_into_jsonl_partitions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/calc/data/convert_into_jsonl_partitions.py -------------------------------------------------------------------------------- /calc/data/tokenize_and_count.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/calc/data/tokenize_and_count.py -------------------------------------------------------------------------------- /imgs/annealing-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/imgs/annealing-example.png -------------------------------------------------------------------------------- /imgs/mamba-moe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/imgs/mamba-moe.png -------------------------------------------------------------------------------- /imgs/mamba.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/imgs/mamba.png -------------------------------------------------------------------------------- /imgs/transformer-moe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/imgs/transformer-moe.png -------------------------------------------------------------------------------- /imgs/transformer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/imgs/transformer.png -------------------------------------------------------------------------------- /imgs/zamba-7b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/imgs/zamba-7b.png -------------------------------------------------------------------------------- /imgs/zamba2-1p2b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/imgs/zamba2-1p2b.png -------------------------------------------------------------------------------- /imgs/zamba2-2p7b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/imgs/zamba2-2p7b.png -------------------------------------------------------------------------------- /imgs/zcookbook.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zyphra/zcookbook/HEAD/imgs/zcookbook.jpg --------------------------------------------------------------------------------