├── .gitignore ├── .gitmodules ├── .pylintrc ├── LICENSE ├── MANIFEST.in ├── Makefile ├── NOTICE ├── README.md ├── README_cn.md ├── csrc ├── .clang-format ├── Makefile ├── clang-lint.sh └── communicators │ ├── nccl_all_gather.cc │ ├── nccl_all_reduce.cc │ ├── nccl_all_to_all.cc │ ├── nccl_broadcast.cc │ ├── nccl_communicator.cc │ ├── nccl_communicator.h │ ├── nccl_reduce.cc │ ├── nccl_reduce_scatter.cc │ └── tensorflow_include │ ├── tensorflow.h │ ├── tensorflow_cuda.h │ └── tensorflow_nccl.h ├── docs ├── .gitignore ├── en │ ├── Makefile │ ├── api │ │ ├── api_examples.md │ │ ├── config.md │ │ ├── index.rst │ │ └── strategy.md │ ├── conf.py │ ├── index.rst │ ├── installation_instructions.md │ ├── overview.md │ ├── quick_start.md │ └── tutorials │ │ ├── ddp.md │ │ ├── index.rst │ │ ├── moe.md │ │ └── pipe.md ├── images │ └── ding-group.png ├── requirements.txt └── zh │ ├── Makefile │ ├── api │ ├── api_examples.md │ ├── config.md │ ├── env.md │ ├── index.rst │ └── strategy.md │ ├── conf.py │ ├── index.rst │ ├── installation_instructions.md │ ├── overview.md │ ├── quick_start.md │ └── tutorials │ ├── ddp.md │ ├── index.rst │ ├── moe.md │ └── pipe.md ├── epl ├── __init__.py ├── cluster.py ├── communicators │ ├── __init__.py │ ├── base.py │ ├── collective_communicator.py │ ├── collective_keys.py │ ├── communication_pool.py │ ├── nccl.py │ ├── nccl_ops.py │ ├── options.py │ ├── pywrap.py │ └── rewriters │ │ ├── __init__.py │ │ ├── base.py │ │ ├── coalescing.py │ │ └── sparse_allreduce.py ├── config.py ├── env.py ├── ir │ ├── __init__.py │ ├── function.py │ ├── graph.py │ ├── layer.py │ ├── op_cluster.py │ ├── operation.py │ ├── phase.py │ ├── shape.py │ ├── sharding_base.py │ ├── taskgraph.py │ ├── tensor.py │ └── variable.py ├── ops │ ├── __init__.py │ ├── adam_weight_decay_optimizer.py │ ├── bridging_layer.py │ ├── distributed_dense.py │ ├── distributed_losses.py │ ├── distributed_ops.py │ └── initializers.py ├── parallel │ ├── __init__.py │ ├── graph_editor.py │ ├── hooks.py │ ├── ops.py │ ├── parallel.py │ ├── partitioner.py │ └── planner.py ├── profiler │ ├── __init__.py │ ├── flops.py │ ├── memory_profiler_hook.py │ └── profiler.py ├── runtime │ ├── __init__.py │ ├── amp │ │ ├── __init__.py │ │ ├── auto_mixed_precision.py │ │ ├── loss_scale.py │ │ └── loss_scale_tf.py │ ├── gc │ │ ├── __init__.py │ │ ├── auto_gradient_checkpoint.py │ │ └── gradient_checkpoint.py │ ├── gradient_accumulation.py │ ├── optimizer_helper.py │ ├── saver.py │ └── zero.py ├── strategies │ ├── __init__.py │ ├── parallel_strategy.py │ ├── replicate.py │ ├── scheduler.py │ ├── split.py │ └── strategy_context.py └── utils │ ├── __init__.py │ ├── common.py │ ├── constant.py │ ├── launcher.py │ ├── metric.py │ ├── shape_inference.py │ ├── summary_info.py │ ├── tf_cflags.py │ ├── tf_ldflags.py │ └── version.py ├── setup.py └── tests ├── .gitignore ├── Makefile ├── add_weights_test.py ├── amp_parallel_test.py ├── amp_test.py ├── auto_cluster_test.py ├── auto_gradient_checkpoint_test.py ├── auto_test.py ├── cluster_test.py ├── cluster_test_with_aware.py ├── cluster_test_with_visible_devices.py ├── collections_test.py ├── communicator_test.py ├── config_env_test.py ├── config_test.py ├── dnn_data_parallel.py ├── eager_test.py ├── env_test.py ├── estimator_dp_example.py ├── estimator_loss_test.py ├── estimator_test.py ├── export_config_env.sh ├── flops_hook_test.py ├── function_test.py ├── gradient_checkpoint_test.py ├── graph_test.py ├── header_test.py ├── hooks_test.py ├── initializers_test.py ├── launch.sh ├── memory_profiler_hook_test.py ├── multi_optimizer_test.py ├── offload_test.py ├── operation_test.py ├── optimizer_helper_test.py ├── optimizer_test.py ├── phase_test.py ├── planner_test.py ├── profiler_test.py ├── run_hook_test.py ├── saver_test.py ├── scheduler_test.py ├── set_default_scope_test.py ├── shape_inference_test.py ├── shape_test.py ├── split_test.py ├── strategy_new_test.py ├── strategy_test.py ├── summary_test.py ├── test_amp_parallel.sh ├── test_launcher.sh ├── test_utils.py ├── utils_test.py ├── while_loop_test.py └── zero_test.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/.gitmodules -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/.pylintrc -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/Makefile -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/NOTICE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/README.md -------------------------------------------------------------------------------- /README_cn.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/README_cn.md -------------------------------------------------------------------------------- /csrc/.clang-format: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/.clang-format -------------------------------------------------------------------------------- /csrc/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/Makefile -------------------------------------------------------------------------------- /csrc/clang-lint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/clang-lint.sh -------------------------------------------------------------------------------- /csrc/communicators/nccl_all_gather.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/communicators/nccl_all_gather.cc -------------------------------------------------------------------------------- /csrc/communicators/nccl_all_reduce.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/communicators/nccl_all_reduce.cc -------------------------------------------------------------------------------- /csrc/communicators/nccl_all_to_all.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/communicators/nccl_all_to_all.cc -------------------------------------------------------------------------------- /csrc/communicators/nccl_broadcast.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/communicators/nccl_broadcast.cc -------------------------------------------------------------------------------- /csrc/communicators/nccl_communicator.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/communicators/nccl_communicator.cc -------------------------------------------------------------------------------- /csrc/communicators/nccl_communicator.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/communicators/nccl_communicator.h -------------------------------------------------------------------------------- /csrc/communicators/nccl_reduce.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/communicators/nccl_reduce.cc -------------------------------------------------------------------------------- /csrc/communicators/nccl_reduce_scatter.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/communicators/nccl_reduce_scatter.cc -------------------------------------------------------------------------------- /csrc/communicators/tensorflow_include/tensorflow.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/communicators/tensorflow_include/tensorflow.h -------------------------------------------------------------------------------- /csrc/communicators/tensorflow_include/tensorflow_cuda.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/communicators/tensorflow_include/tensorflow_cuda.h -------------------------------------------------------------------------------- /csrc/communicators/tensorflow_include/tensorflow_nccl.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/csrc/communicators/tensorflow_include/tensorflow_nccl.h -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | make.bat 2 | _build 3 | -------------------------------------------------------------------------------- /docs/en/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/Makefile -------------------------------------------------------------------------------- /docs/en/api/api_examples.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/api/api_examples.md -------------------------------------------------------------------------------- /docs/en/api/config.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/api/config.md -------------------------------------------------------------------------------- /docs/en/api/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/api/index.rst -------------------------------------------------------------------------------- /docs/en/api/strategy.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/api/strategy.md -------------------------------------------------------------------------------- /docs/en/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/conf.py -------------------------------------------------------------------------------- /docs/en/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/index.rst -------------------------------------------------------------------------------- /docs/en/installation_instructions.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/installation_instructions.md -------------------------------------------------------------------------------- /docs/en/overview.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/overview.md -------------------------------------------------------------------------------- /docs/en/quick_start.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/quick_start.md -------------------------------------------------------------------------------- /docs/en/tutorials/ddp.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/tutorials/ddp.md -------------------------------------------------------------------------------- /docs/en/tutorials/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/tutorials/index.rst -------------------------------------------------------------------------------- /docs/en/tutorials/moe.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/tutorials/moe.md -------------------------------------------------------------------------------- /docs/en/tutorials/pipe.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/en/tutorials/pipe.md -------------------------------------------------------------------------------- /docs/images/ding-group.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/images/ding-group.png -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/requirements.txt -------------------------------------------------------------------------------- /docs/zh/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/Makefile -------------------------------------------------------------------------------- /docs/zh/api/api_examples.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/api/api_examples.md -------------------------------------------------------------------------------- /docs/zh/api/config.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/api/config.md -------------------------------------------------------------------------------- /docs/zh/api/env.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/api/env.md -------------------------------------------------------------------------------- /docs/zh/api/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/api/index.rst -------------------------------------------------------------------------------- /docs/zh/api/strategy.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/api/strategy.md -------------------------------------------------------------------------------- /docs/zh/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/conf.py -------------------------------------------------------------------------------- /docs/zh/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/index.rst -------------------------------------------------------------------------------- /docs/zh/installation_instructions.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/installation_instructions.md -------------------------------------------------------------------------------- /docs/zh/overview.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/overview.md -------------------------------------------------------------------------------- /docs/zh/quick_start.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/quick_start.md -------------------------------------------------------------------------------- /docs/zh/tutorials/ddp.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/tutorials/ddp.md -------------------------------------------------------------------------------- /docs/zh/tutorials/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/tutorials/index.rst -------------------------------------------------------------------------------- /docs/zh/tutorials/moe.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/tutorials/moe.md -------------------------------------------------------------------------------- /docs/zh/tutorials/pipe.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/docs/zh/tutorials/pipe.md -------------------------------------------------------------------------------- /epl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/__init__.py -------------------------------------------------------------------------------- /epl/cluster.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/cluster.py -------------------------------------------------------------------------------- /epl/communicators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/__init__.py -------------------------------------------------------------------------------- /epl/communicators/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/base.py -------------------------------------------------------------------------------- /epl/communicators/collective_communicator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/collective_communicator.py -------------------------------------------------------------------------------- /epl/communicators/collective_keys.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/collective_keys.py -------------------------------------------------------------------------------- /epl/communicators/communication_pool.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/communication_pool.py -------------------------------------------------------------------------------- /epl/communicators/nccl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/nccl.py -------------------------------------------------------------------------------- /epl/communicators/nccl_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/nccl_ops.py -------------------------------------------------------------------------------- /epl/communicators/options.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/options.py -------------------------------------------------------------------------------- /epl/communicators/pywrap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/pywrap.py -------------------------------------------------------------------------------- /epl/communicators/rewriters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/rewriters/__init__.py -------------------------------------------------------------------------------- /epl/communicators/rewriters/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/rewriters/base.py -------------------------------------------------------------------------------- /epl/communicators/rewriters/coalescing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/rewriters/coalescing.py -------------------------------------------------------------------------------- /epl/communicators/rewriters/sparse_allreduce.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/communicators/rewriters/sparse_allreduce.py -------------------------------------------------------------------------------- /epl/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/config.py -------------------------------------------------------------------------------- /epl/env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/env.py -------------------------------------------------------------------------------- /epl/ir/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/__init__.py -------------------------------------------------------------------------------- /epl/ir/function.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/function.py -------------------------------------------------------------------------------- /epl/ir/graph.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/graph.py -------------------------------------------------------------------------------- /epl/ir/layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/layer.py -------------------------------------------------------------------------------- /epl/ir/op_cluster.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/op_cluster.py -------------------------------------------------------------------------------- /epl/ir/operation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/operation.py -------------------------------------------------------------------------------- /epl/ir/phase.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/phase.py -------------------------------------------------------------------------------- /epl/ir/shape.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/shape.py -------------------------------------------------------------------------------- /epl/ir/sharding_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/sharding_base.py -------------------------------------------------------------------------------- /epl/ir/taskgraph.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/taskgraph.py -------------------------------------------------------------------------------- /epl/ir/tensor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/tensor.py -------------------------------------------------------------------------------- /epl/ir/variable.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ir/variable.py -------------------------------------------------------------------------------- /epl/ops/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ops/__init__.py -------------------------------------------------------------------------------- /epl/ops/adam_weight_decay_optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ops/adam_weight_decay_optimizer.py -------------------------------------------------------------------------------- /epl/ops/bridging_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ops/bridging_layer.py -------------------------------------------------------------------------------- /epl/ops/distributed_dense.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ops/distributed_dense.py -------------------------------------------------------------------------------- /epl/ops/distributed_losses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ops/distributed_losses.py -------------------------------------------------------------------------------- /epl/ops/distributed_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ops/distributed_ops.py -------------------------------------------------------------------------------- /epl/ops/initializers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/ops/initializers.py -------------------------------------------------------------------------------- /epl/parallel/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/parallel/__init__.py -------------------------------------------------------------------------------- /epl/parallel/graph_editor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/parallel/graph_editor.py -------------------------------------------------------------------------------- /epl/parallel/hooks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/parallel/hooks.py -------------------------------------------------------------------------------- /epl/parallel/ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/parallel/ops.py -------------------------------------------------------------------------------- /epl/parallel/parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/parallel/parallel.py -------------------------------------------------------------------------------- /epl/parallel/partitioner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/parallel/partitioner.py -------------------------------------------------------------------------------- /epl/parallel/planner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/parallel/planner.py -------------------------------------------------------------------------------- /epl/profiler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/profiler/__init__.py -------------------------------------------------------------------------------- /epl/profiler/flops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/profiler/flops.py -------------------------------------------------------------------------------- /epl/profiler/memory_profiler_hook.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/profiler/memory_profiler_hook.py -------------------------------------------------------------------------------- /epl/profiler/profiler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/profiler/profiler.py -------------------------------------------------------------------------------- /epl/runtime/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/__init__.py -------------------------------------------------------------------------------- /epl/runtime/amp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/amp/__init__.py -------------------------------------------------------------------------------- /epl/runtime/amp/auto_mixed_precision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/amp/auto_mixed_precision.py -------------------------------------------------------------------------------- /epl/runtime/amp/loss_scale.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/amp/loss_scale.py -------------------------------------------------------------------------------- /epl/runtime/amp/loss_scale_tf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/amp/loss_scale_tf.py -------------------------------------------------------------------------------- /epl/runtime/gc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/gc/__init__.py -------------------------------------------------------------------------------- /epl/runtime/gc/auto_gradient_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/gc/auto_gradient_checkpoint.py -------------------------------------------------------------------------------- /epl/runtime/gc/gradient_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/gc/gradient_checkpoint.py -------------------------------------------------------------------------------- /epl/runtime/gradient_accumulation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/gradient_accumulation.py -------------------------------------------------------------------------------- /epl/runtime/optimizer_helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/optimizer_helper.py -------------------------------------------------------------------------------- /epl/runtime/saver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/saver.py -------------------------------------------------------------------------------- /epl/runtime/zero.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/runtime/zero.py -------------------------------------------------------------------------------- /epl/strategies/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/strategies/__init__.py -------------------------------------------------------------------------------- /epl/strategies/parallel_strategy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/strategies/parallel_strategy.py -------------------------------------------------------------------------------- /epl/strategies/replicate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/strategies/replicate.py -------------------------------------------------------------------------------- /epl/strategies/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/strategies/scheduler.py -------------------------------------------------------------------------------- /epl/strategies/split.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/strategies/split.py -------------------------------------------------------------------------------- /epl/strategies/strategy_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/strategies/strategy_context.py -------------------------------------------------------------------------------- /epl/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/utils/__init__.py -------------------------------------------------------------------------------- /epl/utils/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/utils/common.py -------------------------------------------------------------------------------- /epl/utils/constant.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/utils/constant.py -------------------------------------------------------------------------------- /epl/utils/launcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/utils/launcher.py -------------------------------------------------------------------------------- /epl/utils/metric.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/utils/metric.py -------------------------------------------------------------------------------- /epl/utils/shape_inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/utils/shape_inference.py -------------------------------------------------------------------------------- /epl/utils/summary_info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/utils/summary_info.py -------------------------------------------------------------------------------- /epl/utils/tf_cflags.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/utils/tf_cflags.py -------------------------------------------------------------------------------- /epl/utils/tf_ldflags.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/utils/tf_ldflags.py -------------------------------------------------------------------------------- /epl/utils/version.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/epl/utils/version.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/setup.py -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/Makefile -------------------------------------------------------------------------------- /tests/add_weights_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/add_weights_test.py -------------------------------------------------------------------------------- /tests/amp_parallel_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/amp_parallel_test.py -------------------------------------------------------------------------------- /tests/amp_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/amp_test.py -------------------------------------------------------------------------------- /tests/auto_cluster_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/auto_cluster_test.py -------------------------------------------------------------------------------- /tests/auto_gradient_checkpoint_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/auto_gradient_checkpoint_test.py -------------------------------------------------------------------------------- /tests/auto_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/auto_test.py -------------------------------------------------------------------------------- /tests/cluster_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/cluster_test.py -------------------------------------------------------------------------------- /tests/cluster_test_with_aware.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/cluster_test_with_aware.py -------------------------------------------------------------------------------- /tests/cluster_test_with_visible_devices.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/cluster_test_with_visible_devices.py -------------------------------------------------------------------------------- /tests/collections_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/collections_test.py -------------------------------------------------------------------------------- /tests/communicator_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/communicator_test.py -------------------------------------------------------------------------------- /tests/config_env_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/config_env_test.py -------------------------------------------------------------------------------- /tests/config_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/config_test.py -------------------------------------------------------------------------------- /tests/dnn_data_parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/dnn_data_parallel.py -------------------------------------------------------------------------------- /tests/eager_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/eager_test.py -------------------------------------------------------------------------------- /tests/env_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/env_test.py -------------------------------------------------------------------------------- /tests/estimator_dp_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/estimator_dp_example.py -------------------------------------------------------------------------------- /tests/estimator_loss_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/estimator_loss_test.py -------------------------------------------------------------------------------- /tests/estimator_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/estimator_test.py -------------------------------------------------------------------------------- /tests/export_config_env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/export_config_env.sh -------------------------------------------------------------------------------- /tests/flops_hook_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/flops_hook_test.py -------------------------------------------------------------------------------- /tests/function_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/function_test.py -------------------------------------------------------------------------------- /tests/gradient_checkpoint_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/gradient_checkpoint_test.py -------------------------------------------------------------------------------- /tests/graph_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/graph_test.py -------------------------------------------------------------------------------- /tests/header_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/header_test.py -------------------------------------------------------------------------------- /tests/hooks_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/hooks_test.py -------------------------------------------------------------------------------- /tests/initializers_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/initializers_test.py -------------------------------------------------------------------------------- /tests/launch.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/launch.sh -------------------------------------------------------------------------------- /tests/memory_profiler_hook_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/memory_profiler_hook_test.py -------------------------------------------------------------------------------- /tests/multi_optimizer_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/multi_optimizer_test.py -------------------------------------------------------------------------------- /tests/offload_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/offload_test.py -------------------------------------------------------------------------------- /tests/operation_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/operation_test.py -------------------------------------------------------------------------------- /tests/optimizer_helper_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/optimizer_helper_test.py -------------------------------------------------------------------------------- /tests/optimizer_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/optimizer_test.py -------------------------------------------------------------------------------- /tests/phase_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/phase_test.py -------------------------------------------------------------------------------- /tests/planner_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/planner_test.py -------------------------------------------------------------------------------- /tests/profiler_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/profiler_test.py -------------------------------------------------------------------------------- /tests/run_hook_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/run_hook_test.py -------------------------------------------------------------------------------- /tests/saver_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/saver_test.py -------------------------------------------------------------------------------- /tests/scheduler_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/scheduler_test.py -------------------------------------------------------------------------------- /tests/set_default_scope_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/set_default_scope_test.py -------------------------------------------------------------------------------- /tests/shape_inference_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/shape_inference_test.py -------------------------------------------------------------------------------- /tests/shape_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/shape_test.py -------------------------------------------------------------------------------- /tests/split_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/split_test.py -------------------------------------------------------------------------------- /tests/strategy_new_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/strategy_new_test.py -------------------------------------------------------------------------------- /tests/strategy_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/strategy_test.py -------------------------------------------------------------------------------- /tests/summary_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/summary_test.py -------------------------------------------------------------------------------- /tests/test_amp_parallel.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/test_amp_parallel.sh -------------------------------------------------------------------------------- /tests/test_launcher.sh: -------------------------------------------------------------------------------- 1 | python dnn_data_parallel.py 2 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/test_utils.py -------------------------------------------------------------------------------- /tests/utils_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/utils_test.py -------------------------------------------------------------------------------- /tests/while_loop_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/while_loop_test.py -------------------------------------------------------------------------------- /tests/zero_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/EasyParallelLibrary/HEAD/tests/zero_test.py --------------------------------------------------------------------------------