├── .github └── workflows │ ├── gpu.yml │ └── ruff.yml ├── .gitignore ├── .gitmodules ├── .pre-commit-config.yaml ├── INTELLECT_1_Technical_Report.pdf ├── LICENSE ├── README.md ├── configs ├── 10B │ ├── H100.toml │ ├── H100_cooldown.toml │ ├── H100_devel.toml │ └── H100_simple.toml ├── 13B │ └── H100.toml ├── 150M │ ├── 3090.toml │ ├── A40.toml │ ├── H100-fast.toml │ └── H100.toml ├── 150M_short │ ├── 3090.toml │ ├── A40.toml │ └── H100.toml ├── 1B │ └── H100.toml ├── 70M │ └── H100.toml ├── 7B │ └── H100.toml ├── 7B_diloco │ └── H100.toml ├── debug │ ├── diloco.toml │ └── normal.toml └── test.toml ├── pyproject.toml ├── scripts ├── all_reduce.py ├── bandwith │ ├── down.sh │ └── up.sh ├── convert_dl_ckpt.sh ├── convert_dl_state.py ├── export_dcp.py ├── install │ └── install.sh ├── simple_gloo.py ├── simulate_multi_node_diloco.sh ├── skip_data.py └── subset_data.py ├── src └── zeroband │ ├── C │ ├── __init__.py │ ├── collectives.py │ ├── compression.py │ └── csrc │ │ ├── collectives.cpp │ │ └── compression.cpp │ ├── __init__.py │ ├── checkpoint.py │ ├── collectives.py │ ├── comms.py │ ├── compression.py │ ├── config.py │ ├── data.py │ ├── diloco.py │ ├── loss.py │ ├── lr_scheduler.py │ ├── models │ ├── __init__.py │ ├── llama │ │ ├── __init__.py │ │ └── model.py │ └── norms.py │ ├── optimizers.py │ ├── train.py │ └── utils │ ├── __init__.py │ ├── activation_ckpt.py │ ├── ip.py │ ├── logger.py │ ├── metric_logger.py │ ├── profiler.py │ ├── state_dict_send_recv.py │ ├── stopwatch.py │ ├── wget.py │ └── world_info.py ├── tests ├── test_c │ ├── conftest.py │ ├── test_collectives.py │ └── test_compression.py ├── test_configs.py ├── test_data.py ├── test_dist │ ├── conftest.py │ ├── test_comms.py │ ├── test_diloco.py │ └── test_send_state_dict.py ├── test_model.py └── test_torchrun │ └── test_train.py └── uv.lock /.github/workflows/gpu.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/.github/workflows/gpu.yml -------------------------------------------------------------------------------- /.github/workflows/ruff.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/.github/workflows/ruff.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/.gitmodules -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /INTELLECT_1_Technical_Report.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/INTELLECT_1_Technical_Report.pdf -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/README.md -------------------------------------------------------------------------------- /configs/10B/H100.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/10B/H100.toml -------------------------------------------------------------------------------- /configs/10B/H100_cooldown.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/10B/H100_cooldown.toml -------------------------------------------------------------------------------- /configs/10B/H100_devel.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/10B/H100_devel.toml -------------------------------------------------------------------------------- /configs/10B/H100_simple.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/10B/H100_simple.toml -------------------------------------------------------------------------------- /configs/13B/H100.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/13B/H100.toml -------------------------------------------------------------------------------- /configs/150M/3090.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/150M/3090.toml -------------------------------------------------------------------------------- /configs/150M/A40.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/150M/A40.toml -------------------------------------------------------------------------------- /configs/150M/H100-fast.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/150M/H100-fast.toml -------------------------------------------------------------------------------- /configs/150M/H100.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/150M/H100.toml -------------------------------------------------------------------------------- /configs/150M_short/3090.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/150M_short/3090.toml -------------------------------------------------------------------------------- /configs/150M_short/A40.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/150M_short/A40.toml -------------------------------------------------------------------------------- /configs/150M_short/H100.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/150M_short/H100.toml -------------------------------------------------------------------------------- /configs/1B/H100.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/1B/H100.toml -------------------------------------------------------------------------------- /configs/70M/H100.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/70M/H100.toml -------------------------------------------------------------------------------- /configs/7B/H100.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/7B/H100.toml -------------------------------------------------------------------------------- /configs/7B_diloco/H100.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/7B_diloco/H100.toml -------------------------------------------------------------------------------- /configs/debug/diloco.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/debug/diloco.toml -------------------------------------------------------------------------------- /configs/debug/normal.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/debug/normal.toml -------------------------------------------------------------------------------- /configs/test.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/configs/test.toml -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/all_reduce.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/scripts/all_reduce.py -------------------------------------------------------------------------------- /scripts/bandwith/down.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/scripts/bandwith/down.sh -------------------------------------------------------------------------------- /scripts/bandwith/up.sh: -------------------------------------------------------------------------------- 1 | tc qdisc del dev lo root 2 | -------------------------------------------------------------------------------- /scripts/convert_dl_ckpt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/scripts/convert_dl_ckpt.sh -------------------------------------------------------------------------------- /scripts/convert_dl_state.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/scripts/convert_dl_state.py -------------------------------------------------------------------------------- /scripts/export_dcp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/scripts/export_dcp.py -------------------------------------------------------------------------------- /scripts/install/install.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/scripts/install/install.sh -------------------------------------------------------------------------------- /scripts/simple_gloo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/scripts/simple_gloo.py -------------------------------------------------------------------------------- /scripts/simulate_multi_node_diloco.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/scripts/simulate_multi_node_diloco.sh -------------------------------------------------------------------------------- /scripts/skip_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/scripts/skip_data.py -------------------------------------------------------------------------------- /scripts/subset_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/scripts/subset_data.py -------------------------------------------------------------------------------- /src/zeroband/C/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/zeroband/C/collectives.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/C/collectives.py -------------------------------------------------------------------------------- /src/zeroband/C/compression.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/C/compression.py -------------------------------------------------------------------------------- /src/zeroband/C/csrc/collectives.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/C/csrc/collectives.cpp -------------------------------------------------------------------------------- /src/zeroband/C/csrc/compression.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/C/csrc/compression.cpp -------------------------------------------------------------------------------- /src/zeroband/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/zeroband/checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/checkpoint.py -------------------------------------------------------------------------------- /src/zeroband/collectives.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/collectives.py -------------------------------------------------------------------------------- /src/zeroband/comms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/comms.py -------------------------------------------------------------------------------- /src/zeroband/compression.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/compression.py -------------------------------------------------------------------------------- /src/zeroband/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/config.py -------------------------------------------------------------------------------- /src/zeroband/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/data.py -------------------------------------------------------------------------------- /src/zeroband/diloco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/diloco.py -------------------------------------------------------------------------------- /src/zeroband/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/loss.py -------------------------------------------------------------------------------- /src/zeroband/lr_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/lr_scheduler.py -------------------------------------------------------------------------------- /src/zeroband/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/zeroband/models/llama/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/models/llama/__init__.py -------------------------------------------------------------------------------- /src/zeroband/models/llama/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/models/llama/model.py -------------------------------------------------------------------------------- /src/zeroband/models/norms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/models/norms.py -------------------------------------------------------------------------------- /src/zeroband/optimizers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/optimizers.py -------------------------------------------------------------------------------- /src/zeroband/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/train.py -------------------------------------------------------------------------------- /src/zeroband/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/utils/__init__.py -------------------------------------------------------------------------------- /src/zeroband/utils/activation_ckpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/utils/activation_ckpt.py -------------------------------------------------------------------------------- /src/zeroband/utils/ip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/utils/ip.py -------------------------------------------------------------------------------- /src/zeroband/utils/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/utils/logger.py -------------------------------------------------------------------------------- /src/zeroband/utils/metric_logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/utils/metric_logger.py -------------------------------------------------------------------------------- /src/zeroband/utils/profiler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/utils/profiler.py -------------------------------------------------------------------------------- /src/zeroband/utils/state_dict_send_recv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/utils/state_dict_send_recv.py -------------------------------------------------------------------------------- /src/zeroband/utils/stopwatch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/utils/stopwatch.py -------------------------------------------------------------------------------- /src/zeroband/utils/wget.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/utils/wget.py -------------------------------------------------------------------------------- /src/zeroband/utils/world_info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/src/zeroband/utils/world_info.py -------------------------------------------------------------------------------- /tests/test_c/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/tests/test_c/conftest.py -------------------------------------------------------------------------------- /tests/test_c/test_collectives.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/tests/test_c/test_collectives.py -------------------------------------------------------------------------------- /tests/test_c/test_compression.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/tests/test_c/test_compression.py -------------------------------------------------------------------------------- /tests/test_configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/tests/test_configs.py -------------------------------------------------------------------------------- /tests/test_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/tests/test_data.py -------------------------------------------------------------------------------- /tests/test_dist/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/tests/test_dist/conftest.py -------------------------------------------------------------------------------- /tests/test_dist/test_comms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/tests/test_dist/test_comms.py -------------------------------------------------------------------------------- /tests/test_dist/test_diloco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/tests/test_dist/test_diloco.py -------------------------------------------------------------------------------- /tests/test_dist/test_send_state_dict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/tests/test_dist/test_send_state_dict.py -------------------------------------------------------------------------------- /tests/test_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/tests/test_model.py -------------------------------------------------------------------------------- /tests/test_torchrun/test_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/tests/test_torchrun/test_train.py -------------------------------------------------------------------------------- /uv.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrimeIntellect-ai/prime-diloco/HEAD/uv.lock --------------------------------------------------------------------------------