├── .gitignore ├── LICENSE ├── README.md ├── convex_code ├── README.md ├── base_logistic.py ├── baselines.py ├── constants.py ├── dump │ └── optimum-epsilon │ │ └── baselines.pickle ├── experiment.py ├── experiment_epsilon_final.py ├── logistic.py ├── parameters.py ├── pickle_datasets.py ├── plots-epsilon.ipynb └── utils.py └── dl_code ├── README.md ├── auto_extract.py ├── environments └── docker │ ├── base │ ├── .screenrc │ ├── .tmux.conf │ ├── Dockerfile │ ├── entrypoint.sh │ └── fix-permissions │ ├── docker-compose.yml │ └── pytorch-mpi │ └── Dockerfile ├── exps ├── finetune-lstm │ └── example.sh ├── finetune-resnet20-cifar10 │ └── example.sh └── finetune-resnet50-imagenet │ ├── decentralized_sgd.sh │ └── example.sh ├── hostfile ├── main.py ├── parameters.py ├── pcode ├── __init__.py ├── create_dataset.py ├── create_metrics.py ├── create_model.py ├── create_optimizer.py ├── create_scheduler.py ├── datasets │ ├── __init__.py │ ├── loader │ │ ├── __init__.py │ │ ├── epsilon_or_rcv1_folder.py │ │ ├── imagenet_folder.py │ │ ├── preprocess_toolkit.py │ │ ├── serialize.py │ │ ├── svhn_folder.py │ │ └── utils.py │ ├── partition_data.py │ └── prepare_data.py ├── distributed_running_cv.py ├── distributed_running_nlp.py ├── models │ ├── __init__.py │ ├── data_parallel_wrapper.py │ ├── densenet.py │ ├── lenet.py │ ├── mlp.py │ ├── resnet.py │ ├── rnn_lm.py │ ├── vgg.py │ └── wideresnet.py ├── optim │ ├── __init__.py │ ├── dcd_psgd.py │ ├── deep_squeeze.py │ ├── dgc.py │ ├── ecd_psgd.py │ ├── ef_sign_sgd.py │ ├── parallel_choco.py │ ├── parallel_choco_v.py │ ├── sgd.py │ └── utils.py ├── tools │ ├── __init__.py │ ├── plot.py │ ├── plot_utils.py │ ├── show_results.py │ └── tune_consensus_stepsize.py └── utils │ ├── __init__.py │ ├── auxiliary.py │ ├── checkpoint.py │ ├── communication.py │ ├── error_handler.py │ ├── logging.py │ ├── mathdict.py │ ├── op_files.py │ ├── op_paths.py │ ├── sparsification.py │ ├── stat_tracker.py │ ├── tensor_buffer.py │ ├── timer.py │ └── topology.py ├── run.py └── tmux_cluster ├── __init__.py ├── tmux.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | *iccluster 2 | *.vscode 3 | *Makefile 4 | *test.sh -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/README.md -------------------------------------------------------------------------------- /convex_code/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/convex_code/README.md -------------------------------------------------------------------------------- /convex_code/base_logistic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/convex_code/base_logistic.py -------------------------------------------------------------------------------- /convex_code/baselines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/convex_code/baselines.py -------------------------------------------------------------------------------- /convex_code/constants.py: -------------------------------------------------------------------------------- 1 | INIT_WEIGHT_STD = 0.01 2 | LOSS_PER_EPOCH = 10 3 | -------------------------------------------------------------------------------- /convex_code/dump/optimum-epsilon/baselines.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/convex_code/dump/optimum-epsilon/baselines.pickle -------------------------------------------------------------------------------- /convex_code/experiment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/convex_code/experiment.py -------------------------------------------------------------------------------- /convex_code/experiment_epsilon_final.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/convex_code/experiment_epsilon_final.py -------------------------------------------------------------------------------- /convex_code/logistic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/convex_code/logistic.py -------------------------------------------------------------------------------- /convex_code/parameters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/convex_code/parameters.py -------------------------------------------------------------------------------- /convex_code/pickle_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/convex_code/pickle_datasets.py -------------------------------------------------------------------------------- /convex_code/plots-epsilon.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/convex_code/plots-epsilon.ipynb -------------------------------------------------------------------------------- /convex_code/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/convex_code/utils.py -------------------------------------------------------------------------------- /dl_code/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/README.md -------------------------------------------------------------------------------- /dl_code/auto_extract.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/auto_extract.py -------------------------------------------------------------------------------- /dl_code/environments/docker/base/.screenrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/environments/docker/base/.screenrc -------------------------------------------------------------------------------- /dl_code/environments/docker/base/.tmux.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/environments/docker/base/.tmux.conf -------------------------------------------------------------------------------- /dl_code/environments/docker/base/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/environments/docker/base/Dockerfile -------------------------------------------------------------------------------- /dl_code/environments/docker/base/entrypoint.sh: -------------------------------------------------------------------------------- 1 | sudo service ssh start 2 | exec "$@" 3 | -------------------------------------------------------------------------------- /dl_code/environments/docker/base/fix-permissions: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/environments/docker/base/fix-permissions -------------------------------------------------------------------------------- /dl_code/environments/docker/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/environments/docker/docker-compose.yml -------------------------------------------------------------------------------- /dl_code/environments/docker/pytorch-mpi/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/environments/docker/pytorch-mpi/Dockerfile -------------------------------------------------------------------------------- /dl_code/exps/finetune-lstm/example.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/exps/finetune-lstm/example.sh -------------------------------------------------------------------------------- /dl_code/exps/finetune-resnet20-cifar10/example.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/exps/finetune-resnet20-cifar10/example.sh -------------------------------------------------------------------------------- /dl_code/exps/finetune-resnet50-imagenet/decentralized_sgd.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/exps/finetune-resnet50-imagenet/decentralized_sgd.sh -------------------------------------------------------------------------------- /dl_code/exps/finetune-resnet50-imagenet/example.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/exps/finetune-resnet50-imagenet/example.sh -------------------------------------------------------------------------------- /dl_code/hostfile: -------------------------------------------------------------------------------- 1 | localhost slots=32 -------------------------------------------------------------------------------- /dl_code/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/main.py -------------------------------------------------------------------------------- /dl_code/parameters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/parameters.py -------------------------------------------------------------------------------- /dl_code/pcode/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dl_code/pcode/create_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/create_dataset.py -------------------------------------------------------------------------------- /dl_code/pcode/create_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/create_metrics.py -------------------------------------------------------------------------------- /dl_code/pcode/create_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/create_model.py -------------------------------------------------------------------------------- /dl_code/pcode/create_optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/create_optimizer.py -------------------------------------------------------------------------------- /dl_code/pcode/create_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/create_scheduler.py -------------------------------------------------------------------------------- /dl_code/pcode/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dl_code/pcode/datasets/loader/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dl_code/pcode/datasets/loader/epsilon_or_rcv1_folder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/datasets/loader/epsilon_or_rcv1_folder.py -------------------------------------------------------------------------------- /dl_code/pcode/datasets/loader/imagenet_folder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/datasets/loader/imagenet_folder.py -------------------------------------------------------------------------------- /dl_code/pcode/datasets/loader/preprocess_toolkit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/datasets/loader/preprocess_toolkit.py -------------------------------------------------------------------------------- /dl_code/pcode/datasets/loader/serialize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/datasets/loader/serialize.py -------------------------------------------------------------------------------- /dl_code/pcode/datasets/loader/svhn_folder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/datasets/loader/svhn_folder.py -------------------------------------------------------------------------------- /dl_code/pcode/datasets/loader/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/datasets/loader/utils.py -------------------------------------------------------------------------------- /dl_code/pcode/datasets/partition_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/datasets/partition_data.py -------------------------------------------------------------------------------- /dl_code/pcode/datasets/prepare_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/datasets/prepare_data.py -------------------------------------------------------------------------------- /dl_code/pcode/distributed_running_cv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/distributed_running_cv.py -------------------------------------------------------------------------------- /dl_code/pcode/distributed_running_nlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/distributed_running_nlp.py -------------------------------------------------------------------------------- /dl_code/pcode/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/models/__init__.py -------------------------------------------------------------------------------- /dl_code/pcode/models/data_parallel_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/models/data_parallel_wrapper.py -------------------------------------------------------------------------------- /dl_code/pcode/models/densenet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/models/densenet.py -------------------------------------------------------------------------------- /dl_code/pcode/models/lenet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/models/lenet.py -------------------------------------------------------------------------------- /dl_code/pcode/models/mlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/models/mlp.py -------------------------------------------------------------------------------- /dl_code/pcode/models/resnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/models/resnet.py -------------------------------------------------------------------------------- /dl_code/pcode/models/rnn_lm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/models/rnn_lm.py -------------------------------------------------------------------------------- /dl_code/pcode/models/vgg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/models/vgg.py -------------------------------------------------------------------------------- /dl_code/pcode/models/wideresnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/models/wideresnet.py -------------------------------------------------------------------------------- /dl_code/pcode/optim/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dl_code/pcode/optim/dcd_psgd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/optim/dcd_psgd.py -------------------------------------------------------------------------------- /dl_code/pcode/optim/deep_squeeze.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/optim/deep_squeeze.py -------------------------------------------------------------------------------- /dl_code/pcode/optim/dgc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/optim/dgc.py -------------------------------------------------------------------------------- /dl_code/pcode/optim/ecd_psgd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/optim/ecd_psgd.py -------------------------------------------------------------------------------- /dl_code/pcode/optim/ef_sign_sgd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/optim/ef_sign_sgd.py -------------------------------------------------------------------------------- /dl_code/pcode/optim/parallel_choco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/optim/parallel_choco.py -------------------------------------------------------------------------------- /dl_code/pcode/optim/parallel_choco_v.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/optim/parallel_choco_v.py -------------------------------------------------------------------------------- /dl_code/pcode/optim/sgd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/optim/sgd.py -------------------------------------------------------------------------------- /dl_code/pcode/optim/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/optim/utils.py -------------------------------------------------------------------------------- /dl_code/pcode/tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dl_code/pcode/tools/plot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/tools/plot.py -------------------------------------------------------------------------------- /dl_code/pcode/tools/plot_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/tools/plot_utils.py -------------------------------------------------------------------------------- /dl_code/pcode/tools/show_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/tools/show_results.py -------------------------------------------------------------------------------- /dl_code/pcode/tools/tune_consensus_stepsize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/tools/tune_consensus_stepsize.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dl_code/pcode/utils/auxiliary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/auxiliary.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/checkpoint.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/communication.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/communication.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/error_handler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/error_handler.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/logging.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/mathdict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/mathdict.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/op_files.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/op_files.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/op_paths.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/op_paths.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/sparsification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/sparsification.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/stat_tracker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/stat_tracker.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/tensor_buffer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/tensor_buffer.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/timer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/timer.py -------------------------------------------------------------------------------- /dl_code/pcode/utils/topology.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/pcode/utils/topology.py -------------------------------------------------------------------------------- /dl_code/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/run.py -------------------------------------------------------------------------------- /dl_code/tmux_cluster/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dl_code/tmux_cluster/tmux.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/tmux_cluster/tmux.py -------------------------------------------------------------------------------- /dl_code/tmux_cluster/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epfml/ChocoSGD/HEAD/dl_code/tmux_cluster/utils.py --------------------------------------------------------------------------------