├── .gitignore ├── Dockerfile ├── README.md ├── docs ├── OWT DiLoCo N=4.png ├── README.md ├── diloco-batchsize.png ├── imagination.png ├── macs.png └── mnist_compare.png ├── example ├── diloco_scaling.py ├── image_train.py ├── mnist.py ├── nanogpt │ ├── __init__.py │ ├── build_dataset.py │ ├── dataset.py │ ├── download_dataset.py │ ├── gpt_dataset.py │ ├── nanogpt.py │ └── shakespeare_inference.py ├── nanogpt_train.py └── playground.py ├── exogym ├── __init__.py ├── aux │ ├── __init__.py │ ├── checkpointing.py │ ├── correlation.py │ ├── local_train.py │ ├── logger.py │ └── utils.py ├── common.py ├── minibatch_probe.py ├── strategy │ ├── __init__.py │ ├── communicate.py │ ├── communicate_optimize_strategy.py │ ├── demo.py │ ├── demo_impl │ │ ├── __init__.py │ │ └── demo.py │ ├── diloco.py │ ├── federated_averaging.py │ ├── optim.py │ ├── sparta.py │ ├── sparta_diloco.py │ └── strategy.py ├── train_node.py ├── trainer.py └── utils.py ├── llms.txt └── pyproject.toml /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/.gitignore -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/Dockerfile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/README.md -------------------------------------------------------------------------------- /docs/OWT DiLoCo N=4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/docs/OWT DiLoCo N=4.png -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/docs/README.md -------------------------------------------------------------------------------- /docs/diloco-batchsize.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/docs/diloco-batchsize.png -------------------------------------------------------------------------------- /docs/imagination.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/docs/imagination.png -------------------------------------------------------------------------------- /docs/macs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/docs/macs.png -------------------------------------------------------------------------------- /docs/mnist_compare.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/docs/mnist_compare.png -------------------------------------------------------------------------------- /example/diloco_scaling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/diloco_scaling.py -------------------------------------------------------------------------------- /example/image_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/image_train.py -------------------------------------------------------------------------------- /example/mnist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/mnist.py -------------------------------------------------------------------------------- /example/nanogpt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/nanogpt/__init__.py -------------------------------------------------------------------------------- /example/nanogpt/build_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/nanogpt/build_dataset.py -------------------------------------------------------------------------------- /example/nanogpt/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/nanogpt/dataset.py -------------------------------------------------------------------------------- /example/nanogpt/download_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/nanogpt/download_dataset.py -------------------------------------------------------------------------------- /example/nanogpt/gpt_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/nanogpt/gpt_dataset.py -------------------------------------------------------------------------------- /example/nanogpt/nanogpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/nanogpt/nanogpt.py -------------------------------------------------------------------------------- /example/nanogpt/shakespeare_inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/nanogpt/shakespeare_inference.py -------------------------------------------------------------------------------- /example/nanogpt_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/nanogpt_train.py -------------------------------------------------------------------------------- /example/playground.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/example/playground.py -------------------------------------------------------------------------------- /exogym/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/__init__.py -------------------------------------------------------------------------------- /exogym/aux/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/aux/__init__.py -------------------------------------------------------------------------------- /exogym/aux/checkpointing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/aux/checkpointing.py -------------------------------------------------------------------------------- /exogym/aux/correlation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/aux/correlation.py -------------------------------------------------------------------------------- /exogym/aux/local_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/aux/local_train.py -------------------------------------------------------------------------------- /exogym/aux/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/aux/logger.py -------------------------------------------------------------------------------- /exogym/aux/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/aux/utils.py -------------------------------------------------------------------------------- /exogym/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/common.py -------------------------------------------------------------------------------- /exogym/minibatch_probe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/minibatch_probe.py -------------------------------------------------------------------------------- /exogym/strategy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/__init__.py -------------------------------------------------------------------------------- /exogym/strategy/communicate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/communicate.py -------------------------------------------------------------------------------- /exogym/strategy/communicate_optimize_strategy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/communicate_optimize_strategy.py -------------------------------------------------------------------------------- /exogym/strategy/demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/demo.py -------------------------------------------------------------------------------- /exogym/strategy/demo_impl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/demo_impl/__init__.py -------------------------------------------------------------------------------- /exogym/strategy/demo_impl/demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/demo_impl/demo.py -------------------------------------------------------------------------------- /exogym/strategy/diloco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/diloco.py -------------------------------------------------------------------------------- /exogym/strategy/federated_averaging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/federated_averaging.py -------------------------------------------------------------------------------- /exogym/strategy/optim.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/optim.py -------------------------------------------------------------------------------- /exogym/strategy/sparta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/sparta.py -------------------------------------------------------------------------------- /exogym/strategy/sparta_diloco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/sparta_diloco.py -------------------------------------------------------------------------------- /exogym/strategy/strategy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/strategy/strategy.py -------------------------------------------------------------------------------- /exogym/train_node.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/train_node.py -------------------------------------------------------------------------------- /exogym/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/trainer.py -------------------------------------------------------------------------------- /exogym/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/exogym/utils.py -------------------------------------------------------------------------------- /llms.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/llms.txt -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exo-explore/gym/HEAD/pyproject.toml --------------------------------------------------------------------------------