├── .gitignore ├── DeepSpeed ├── .clang-format ├── .gitignore ├── .pre-commit-config.yaml ├── .pylintrc ├── .readthedocs.yml ├── .style.yapf ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── MANIFEST_win.in ├── README.md ├── SECURITY.md ├── azure │ └── README.md ├── benchmarks │ ├── __init__.py │ ├── communication │ │ ├── README.md │ │ ├── __init__.py │ │ ├── all_gather.py │ │ ├── all_reduce.py │ │ ├── all_to_all.py │ │ ├── broadcast.py │ │ ├── constants.py │ │ ├── pt2pt.py │ │ ├── run_all.py │ │ └── utils.py │ └── inference │ │ ├── bert-bench.py │ │ ├── collect_results.py │ │ ├── gpt-bench.py │ │ ├── requirements.txt │ │ ├── run_model.sh │ │ └── sweep.sh ├── bin │ ├── deepspeed │ ├── deepspeed.pt │ ├── ds │ ├── ds_bench │ ├── ds_elastic │ ├── ds_report │ ├── ds_ssh │ └── dsr ├── csrc │ ├── adagrad │ │ └── cpu_adagrad.cpp │ ├── adam │ │ ├── cpu_adam.cpp │ │ ├── fused_adam_frontend.cpp │ │ ├── multi_tensor_adam.cu │ │ └── multi_tensor_apply.cuh │ ├── aio │ │ ├── common │ │ │ ├── deepspeed_aio_common.cpp │ │ │ ├── deepspeed_aio_common.h │ │ │ ├── deepspeed_aio_types.cpp │ │ │ ├── deepspeed_aio_types.h │ │ │ ├── deepspeed_aio_utils.cpp │ │ │ └── deepspeed_aio_utils.h │ │ ├── py_lib │ │ │ ├── deepspeed_aio_thread.cpp │ │ │ ├── deepspeed_aio_thread.h │ │ │ ├── deepspeed_py_aio.cpp │ │ │ ├── deepspeed_py_aio.h │ │ │ ├── deepspeed_py_aio_handle.cpp │ │ │ ├── deepspeed_py_aio_handle.h │ │ │ ├── deepspeed_py_copy.cpp │ │ │ ├── deepspeed_py_copy.h │ │ │ └── py_ds_aio.cpp │ │ └── py_test │ │ │ ├── aio_bench_generate_param.py │ │ │ ├── aio_bench_perf_sweep.py │ │ │ ├── ds_aio_basic.py │ │ │ ├── ds_aio_handle.py │ │ │ ├── parse_aio_stats.py │ │ │ ├── perf_sweep_utils.py │ │ │ ├── run_read_sweep.sh │ │ │ ├── run_write_sweep.sh │ │ │ ├── single_process_config.json │ │ │ ├── test_ds_aio.py │ │ │ ├── test_ds_aio_utils.py │ │ │ └── validate_async_io.py │ ├── common │ │ └── custom_cuda_kernel.cu │ ├── includes │ │ ├── StopWatch.h │ │ ├── Timer.h │ │ ├── compat.h │ │ ├── context.h │ │ ├── conversion_utils.h │ │ ├── cpu_adagrad.h │ │ ├── cpu_adam.h │ │ ├── cublas_wrappers.h │ │ ├── custom_cuda_layers.h │ │ ├── dropout.h │ │ ├── ds_kernel_utils.h │ │ ├── ds_transformer_cuda.h │ │ ├── feed_forward.h │ │ ├── gelu.h │ │ ├── gemm_test.h │ │ ├── general_kernels.h │ │ ├── memory_access_utils.h │ │ ├── normalize_layer.h │ │ ├── quantizer.h │ │ ├── simd.h │ │ ├── softmax.h │ │ ├── strided_batch_gemm.h │ │ └── type_shim.h │ ├── lamb │ │ ├── fused_lamb_cuda.cpp │ │ └── fused_lamb_cuda_kernel.cu │ ├── quantization │ │ ├── pt_binding.cpp │ │ └── quantizer.cu │ ├── sparse_attention │ │ └── utils.cpp │ ├── transformer │ │ ├── cublas_wrappers.cu │ │ ├── dropout_kernels.cu │ │ ├── ds_transformer_cuda.cpp │ │ ├── gelu_kernels.cu │ │ ├── general_kernels.cu │ │ ├── inference │ │ │ ├── csrc │ │ │ │ ├── apply_rotary_pos_emb.cu │ │ │ │ ├── dequantize.cu │ │ │ │ ├── gelu.cu │ │ │ │ ├── normalize.cu │ │ │ │ ├── pt_binding.cpp │ │ │ │ ├── relu.cu │ │ │ │ ├── softmax.cu │ │ │ │ └── transform.cu │ │ │ └── includes │ │ │ │ ├── inference_context.h │ │ │ │ ├── inference_cublas_wrappers.h │ │ │ │ └── inference_cuda_layers.h │ │ ├── normalize_kernels.cu │ │ ├── softmax_kernels.cu │ │ └── transform_kernels.cu │ └── utils │ │ └── flatten_unflatten.cpp ├── deepspeed │ ├── __init__.py │ ├── autotuning │ │ ├── .gitignore │ │ ├── README.md │ │ ├── __init__.py │ │ ├── autotuner.py │ │ ├── config.py │ │ ├── config_templates │ │ │ ├── template_zero0.json │ │ │ ├── template_zero1.json │ │ │ ├── template_zero2.json │ │ │ └── template_zero3.json │ │ ├── constants.py │ │ ├── scheduler.py │ │ ├── tuner │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── base_tuner.py │ │ │ ├── cost_model.py │ │ │ ├── index_based_tuner.py │ │ │ ├── model_based_tuner.py │ │ │ └── utils.py │ │ └── utils.py │ ├── checkpoint │ │ ├── __init__.py │ │ ├── constants.py │ │ ├── deepspeed_checkpoint.py │ │ ├── reshape_3d_utils.py │ │ ├── reshape_meg_2d.py │ │ ├── reshape_utils.py │ │ ├── universal_checkpoint.py │ │ ├── utils.py │ │ └── zero_checkpoint.py │ ├── comm │ │ ├── __init__.py │ │ ├── backend.py │ │ ├── comm.py │ │ ├── config.py │ │ ├── constants.py │ │ ├── torch.py │ │ └── utils.py │ ├── compression │ │ ├── __init__.py │ │ ├── basic_layer.py │ │ ├── compress.py │ │ ├── config.py │ │ ├── constants.py │ │ ├── helper.py │ │ ├── scheduler.py │ │ └── utils.py │ ├── constants.py │ ├── elasticity │ │ ├── __init__.py │ │ ├── config.py │ │ ├── constants.py │ │ ├── elastic_agent.py │ │ ├── elasticity.py │ │ └── utils.py │ ├── env_report.py │ ├── git_version_info.py │ ├── inference │ │ ├── __init__.py │ │ └── engine.py │ ├── launcher │ │ ├── __init__.py │ │ ├── constants.py │ │ ├── launch.py │ │ ├── multinode_runner.py │ │ └── runner.py │ ├── module_inject │ │ ├── __init__.py │ │ ├── encoder.py │ │ ├── inject.py │ │ ├── layers.py │ │ ├── load_checkpoint.py │ │ ├── module_quantize.py │ │ ├── replace_module.py │ │ ├── replace_policy.py │ │ └── unet.py │ ├── moe │ │ ├── __init__.py │ │ ├── experts.py │ │ ├── layer.py │ │ ├── mappings.py │ │ ├── sharded_moe.py │ │ └── utils.py │ ├── monitor │ │ ├── __init__.py │ │ ├── config.py │ │ ├── constants.py │ │ ├── csv_monitor.py │ │ ├── monitor.py │ │ ├── tensorboard.py │ │ ├── utils.py │ │ └── wandb.py │ ├── nebula │ │ ├── __init__.py │ │ ├── config.py │ │ └── constants.py │ ├── ops │ │ ├── __init__.py │ │ ├── adagrad │ │ │ ├── __init__.py │ │ │ └── cpu_adagrad.py │ │ ├── adam │ │ │ ├── __init__.py │ │ │ ├── cpu_adam.py │ │ │ ├── fused_adam.py │ │ │ └── multi_tensor_apply.py │ │ ├── aio │ │ │ └── __init__.py │ │ ├── csrc │ │ │ ├── adagrad │ │ │ │ └── cpu_adagrad.cpp │ │ │ ├── adam │ │ │ │ ├── cpu_adam.cpp │ │ │ │ ├── fused_adam_frontend.cpp │ │ │ │ ├── multi_tensor_adam.cu │ │ │ │ └── multi_tensor_apply.cuh │ │ │ ├── aio │ │ │ │ ├── common │ │ │ │ │ ├── deepspeed_aio_common.cpp │ │ │ │ │ ├── deepspeed_aio_common.h │ │ │ │ │ ├── deepspeed_aio_types.cpp │ │ │ │ │ ├── deepspeed_aio_types.h │ │ │ │ │ ├── deepspeed_aio_utils.cpp │ │ │ │ │ └── deepspeed_aio_utils.h │ │ │ │ ├── py_lib │ │ │ │ │ ├── deepspeed_aio_thread.cpp │ │ │ │ │ ├── deepspeed_aio_thread.h │ │ │ │ │ ├── deepspeed_py_aio.cpp │ │ │ │ │ ├── deepspeed_py_aio.h │ │ │ │ │ ├── deepspeed_py_aio_handle.cpp │ │ │ │ │ ├── deepspeed_py_aio_handle.h │ │ │ │ │ ├── deepspeed_py_copy.cpp │ │ │ │ │ ├── deepspeed_py_copy.h │ │ │ │ │ └── py_ds_aio.cpp │ │ │ │ └── py_test │ │ │ │ │ ├── aio_bench_generate_param.py │ │ │ │ │ ├── aio_bench_perf_sweep.py │ │ │ │ │ ├── ds_aio_basic.py │ │ │ │ │ ├── ds_aio_handle.py │ │ │ │ │ ├── parse_aio_stats.py │ │ │ │ │ ├── perf_sweep_utils.py │ │ │ │ │ ├── run_read_sweep.sh │ │ │ │ │ ├── run_write_sweep.sh │ │ │ │ │ ├── single_process_config.json │ │ │ │ │ ├── test_ds_aio.py │ │ │ │ │ ├── test_ds_aio_utils.py │ │ │ │ │ └── validate_async_io.py │ │ │ ├── common │ │ │ │ └── custom_cuda_kernel.cu │ │ │ ├── includes │ │ │ │ ├── StopWatch.h │ │ │ │ ├── Timer.h │ │ │ │ ├── compat.h │ │ │ │ ├── context.h │ │ │ │ ├── conversion_utils.h │ │ │ │ ├── cpu_adagrad.h │ │ │ │ ├── cpu_adam.h │ │ │ │ ├── cublas_wrappers.h │ │ │ │ ├── custom_cuda_layers.h │ │ │ │ ├── dropout.h │ │ │ │ ├── ds_kernel_utils.h │ │ │ │ ├── ds_transformer_cuda.h │ │ │ │ ├── feed_forward.h │ │ │ │ ├── gelu.h │ │ │ │ ├── gemm_test.h │ │ │ │ ├── general_kernels.h │ │ │ │ ├── memory_access_utils.h │ │ │ │ ├── normalize_layer.h │ │ │ │ ├── quantizer.h │ │ │ │ ├── simd.h │ │ │ │ ├── softmax.h │ │ │ │ ├── strided_batch_gemm.h │ │ │ │ └── type_shim.h │ │ │ ├── lamb │ │ │ │ ├── fused_lamb_cuda.cpp │ │ │ │ └── fused_lamb_cuda_kernel.cu │ │ │ ├── quantization │ │ │ │ ├── pt_binding.cpp │ │ │ │ └── quantizer.cu │ │ │ ├── sparse_attention │ │ │ │ └── utils.cpp │ │ │ ├── transformer │ │ │ │ ├── cublas_wrappers.cu │ │ │ │ ├── dropout_kernels.cu │ │ │ │ ├── ds_transformer_cuda.cpp │ │ │ │ ├── gelu_kernels.cu │ │ │ │ ├── general_kernels.cu │ │ │ │ ├── inference │ │ │ │ │ ├── csrc │ │ │ │ │ │ ├── apply_rotary_pos_emb.cu │ │ │ │ │ │ ├── dequantize.cu │ │ │ │ │ │ ├── gelu.cu │ │ │ │ │ │ ├── normalize.cu │ │ │ │ │ │ ├── pt_binding.cpp │ │ │ │ │ │ ├── relu.cu │ │ │ │ │ │ ├── softmax.cu │ │ │ │ │ │ └── transform.cu │ │ │ │ │ └── includes │ │ │ │ │ │ ├── inference_context.h │ │ │ │ │ │ ├── inference_cublas_wrappers.h │ │ │ │ │ │ └── inference_cuda_layers.h │ │ │ │ ├── normalize_kernels.cu │ │ │ │ ├── softmax_kernels.cu │ │ │ │ └── transform_kernels.cu │ │ │ └── utils │ │ │ │ └── flatten_unflatten.cpp │ │ ├── lamb │ │ │ ├── __init__.py │ │ │ └── fused_lamb.py │ │ ├── op_builder │ │ │ ├── __init__.py │ │ │ ├── async_io.py │ │ │ ├── builder.py │ │ │ ├── cpu_adagrad.py │ │ │ ├── cpu_adam.py │ │ │ ├── fused_adam.py │ │ │ ├── fused_lamb.py │ │ │ ├── quantizer.py │ │ │ ├── sparse_attn.py │ │ │ ├── stochastic_transformer.py │ │ │ ├── transformer.py │ │ │ ├── transformer_inference.py │ │ │ └── utils.py │ │ ├── quantizer │ │ │ ├── __init__.py │ │ │ └── quantizer.py │ │ ├── sparse_attention │ │ │ ├── __init__.py │ │ │ ├── bert_sparse_self_attention.py │ │ │ ├── matmul.py │ │ │ ├── softmax.py │ │ │ ├── sparse_attention_utils.py │ │ │ ├── sparse_self_attention.py │ │ │ ├── sparsity_config.py │ │ │ └── trsrc │ │ │ │ ├── __init__.py │ │ │ │ ├── matmul.tr │ │ │ │ ├── softmax_bwd.tr │ │ │ │ └── softmax_fwd.tr │ │ └── transformer │ │ │ ├── __init__.py │ │ │ ├── inference │ │ │ ├── __init__.py │ │ │ ├── attention.py │ │ │ ├── moe_inference.py │ │ │ ├── transformer_inference.py │ │ │ └── triton_ops.py │ │ │ └── transformer.py │ ├── pipe │ │ └── __init__.py │ ├── profiling │ │ ├── __init__.py │ │ ├── config.py │ │ ├── constants.py │ │ ├── energy_profiler │ │ │ ├── __init__.py │ │ │ └── profiler.py │ │ └── flops_profiler │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ └── profiler.py │ ├── runtime │ │ ├── __init__.py │ │ ├── activation_checkpointing │ │ │ ├── __init__.py │ │ │ ├── checkpointing.py │ │ │ └── config.py │ │ ├── bf16_optimizer.py │ │ ├── checkpoint_engine │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── checkpoint_engine.py │ │ │ ├── nebula_checkpoint_engine.py │ │ │ └── torch_checkpoint_engine.py │ │ ├── comm │ │ │ ├── __init__.py │ │ │ ├── coalesced_collectives.py │ │ │ ├── mpi.py │ │ │ └── nccl.py │ │ ├── compression │ │ │ ├── __init__.py │ │ │ └── cupy.py │ │ ├── config.py │ │ ├── config_utils.py │ │ ├── constants.py │ │ ├── data_pipeline │ │ │ ├── __init__.py │ │ │ └── curriculum_scheduler.py │ │ ├── dataloader.py │ │ ├── eigenvalue.py │ │ ├── engine.py │ │ ├── fp16 │ │ │ ├── __init__.py │ │ │ ├── fused_optimizer.py │ │ │ ├── loss_scaler.py │ │ │ ├── onebit │ │ │ │ ├── __init__.py │ │ │ │ ├── adam.py │ │ │ │ ├── lamb.py │ │ │ │ └── zoadam.py │ │ │ └── unfused_optimizer.py │ │ ├── lr_schedules.py │ │ ├── pipe │ │ │ ├── __init__.py │ │ │ ├── engine.py │ │ │ ├── module.py │ │ │ ├── p2p.py │ │ │ ├── reconfiguration.py │ │ │ ├── schedule.py │ │ │ └── topology.py │ │ ├── progressive_layer_drop.py │ │ ├── quantize.py │ │ ├── sparse_tensor.py │ │ ├── state_dict_factory.py │ │ ├── swap_tensor │ │ │ ├── __init__.py │ │ │ ├── aio_config.py │ │ │ ├── async_swapper.py │ │ │ ├── constants.py │ │ │ ├── optimizer_utils.py │ │ │ ├── partitioned_optimizer_swapper.py │ │ │ ├── partitioned_param_swapper.py │ │ │ ├── pipelined_optimizer_swapper.py │ │ │ └── utils.py │ │ ├── utils.py │ │ ├── weight_quantizer.py │ │ └── zero │ │ │ ├── __init__.py │ │ │ ├── config.py │ │ │ ├── contiguous_memory_allocator.py │ │ │ ├── linear.py │ │ │ ├── offload_config.py │ │ │ ├── parameter_offload.py │ │ │ ├── partition_parameters.py │ │ │ ├── partitioned_param_coordinator.py │ │ │ ├── stage3.py │ │ │ ├── stage_1_and_2.py │ │ │ ├── test.py │ │ │ ├── tiling.py │ │ │ └── utils.py │ └── utils │ │ ├── __init__.py │ │ ├── comms_logging.py │ │ ├── debug.py │ │ ├── exceptions.py │ │ ├── groups.py │ │ ├── init_on_device.py │ │ ├── logging.py │ │ ├── mixed_precision_linkage.py │ │ ├── nvtx.py │ │ ├── tensor_fragment.py │ │ ├── timer.py │ │ ├── types.py │ │ └── zero_to_fp32.py ├── docker │ ├── Dockerfile │ └── Dockerfile.rocm ├── docs │ ├── 404.html │ ├── CNAME │ ├── Gemfile │ ├── README.md │ ├── _config.yml │ ├── _data │ │ └── navigation.yml │ ├── _includes │ │ ├── analytics.html │ │ ├── archive-single.html │ │ ├── author-profile-custom-links.html │ │ ├── author-profile.html │ │ ├── breadcrumbs.html │ │ ├── browser-upgrade.html │ │ ├── category-list.html │ │ ├── comment.html │ │ ├── comments.html │ │ ├── documents-collection.html │ │ ├── feature_row │ │ ├── figure │ │ ├── footer.html │ │ ├── gallery │ │ ├── group-by-array │ │ ├── head.html │ │ ├── masthead.html │ │ ├── nav_list │ │ ├── page__date.html │ │ ├── page__hero.html │ │ ├── page__hero_video.html │ │ ├── page__meta.html │ │ ├── page__taxonomy.html │ │ ├── paginator.html │ │ ├── post_pagination.html │ │ ├── posts-category.html │ │ ├── posts-tag.html │ │ ├── scripts.html │ │ ├── seo.html │ │ ├── sidebar.html │ │ ├── skip-links.html │ │ ├── social-share.html │ │ ├── tag-list.html │ │ ├── toc │ │ ├── toc.html │ │ └── video │ ├── _layouts │ │ └── single-full.html │ ├── _pages │ │ ├── compression.md │ │ ├── config-json.md │ │ ├── inference.md │ │ ├── posts-landing.md │ │ ├── posts_list_landing.md │ │ ├── training.md │ │ └── tutorials-landing.md │ ├── _posts │ │ ├── 2020-02-13-release.md │ │ ├── 2020-02-13-turing-nlg.md │ │ ├── 2020-03-17-reduce-scatter.md │ │ ├── 2020-05-19-bert-record.md │ │ ├── 2020-05-19-press-release.md │ │ ├── 2020-05-19-zero-stage2.md │ │ ├── 2020-05-28-fastest-bert-training.md │ │ ├── 2020-07-24-deepspeed-webinar.md │ │ ├── 2020-08-07-webinar-on-demand.md │ │ ├── 2020-09-08-sparse-attention-news.md │ │ ├── 2020-09-09-ZeRO-Offload.md │ │ ├── 2020-09-09-onebit-adam-blog-post.md │ │ ├── 2020-09-09-onebit-adam-news.md │ │ ├── 2020-09-09-pipeline-parallelism.md │ │ ├── 2020-09-09-sparse-attention.md │ │ ├── 2020-10-28-progressive-layer-dropping-news.md │ │ ├── 2021-03-08-zero3-offload.md │ │ ├── 2021-05-05-MoQ.md │ │ ├── 2021-05-05-inference-kernel-optimization.md │ │ ├── 2021-05-14-inference-release.md │ │ ├── 2021-08-18-deepspeed-moe.md │ │ ├── 2021-11-15-autotuning.md │ │ ├── 2021-12-09-deepspeed-moe-nlg.md │ │ ├── 2022-01-19-moe-inference.md │ │ ├── 2022-03-21-amd-support.md │ │ ├── 2022-07-26-deepspeed-azure.md │ │ ├── 2022-09-10-zero-inference.md │ │ └── 2022-10-11-mii.md │ ├── _sass │ │ ├── button-group.scss │ │ ├── minimal-mistakes.scss │ │ └── minimal-mistakes │ │ │ ├── _archive.scss │ │ │ ├── _navigation.scss │ │ │ ├── _page.scss │ │ │ ├── _sidebar.scss │ │ │ ├── _variables.scss │ │ │ └── skins │ │ │ └── _air.scss │ ├── _tutorials │ │ ├── MoQ-tutorial.md │ │ ├── advanced-install.md │ │ ├── autotuning.md │ │ ├── azure.md │ │ ├── bert-finetuning.md │ │ ├── bert-pretraining.md │ │ ├── cifar-10.md │ │ ├── comms-logging.md │ │ ├── curriculum-learning.md │ │ ├── flops-profiler.md │ │ ├── gan.md │ │ ├── getting-started.md │ │ ├── inference-tutorial.md │ │ ├── large-models-w-deepspeed.md │ │ ├── lrrt.md │ │ ├── megatron.md │ │ ├── mixture-of-experts-inference.md │ │ ├── mixture-of-experts-nlg.md │ │ ├── mixture-of-experts.md │ │ ├── model-compression.md │ │ ├── monitor.md │ │ ├── one-cycle.md │ │ ├── onebit-adam.md │ │ ├── onebit-lamb.md │ │ ├── pipeline.md │ │ ├── progressive_layer_dropping.md │ │ ├── pytorch-profiler.md │ │ ├── sparse-attention.md │ │ ├── transformer_kernel.md │ │ ├── zero-offload.md │ │ ├── zero-one-adam.md │ │ └── zero.md │ ├── assets │ │ ├── css │ │ │ └── main.scss │ │ └── images │ │ │ ├── 1.3B-MoE-128.png │ │ │ ├── 175b-trend.png │ │ │ ├── 1cycle_lr.png │ │ │ ├── 1t-trend.png │ │ │ ├── 3d-parallelism.png │ │ │ ├── 3pillars.png │ │ │ ├── 530b-trend.png │ │ │ ├── DeepSpeed-vs-Megatron.png │ │ │ ├── DeepSpeed_dark.svg │ │ │ ├── DeepSpeed_dark_transparent.svg │ │ │ ├── DeepSpeed_light.svg │ │ │ ├── DeepSpeed_light_transparent.svg │ │ │ ├── accelerate-dark.png │ │ │ ├── accelerate-light.png │ │ │ ├── accelerate.png │ │ │ ├── adam-convergence.png │ │ │ ├── bert-ib.png │ │ │ ├── bert-large-training-time.png │ │ │ ├── bert-scaling.png │ │ │ ├── bert-tcp.png │ │ │ ├── bingbert-mixedbit.png │ │ │ ├── convergence-table.png │ │ │ ├── deepspeed-logo-uppercase-bold-white-1.15.svg │ │ │ ├── deepspeed-logo-uppercase-bold-white.svg │ │ │ ├── deepspeed-logo-uppercase-white.svg │ │ │ ├── deepspeed-speedup.png │ │ │ ├── end-to-end-bert-training.PNG │ │ │ ├── gpu-numbers.png │ │ │ ├── hf-logo.png │ │ │ ├── hf-transformers.png │ │ │ ├── inference-gemm-scheduling.png │ │ │ ├── inference-kernel-fusion.png │ │ │ ├── inference-latency.png │ │ │ ├── inference-throughput.png │ │ │ ├── large-model-graph.png │ │ │ ├── layernorm_animation.gif │ │ │ ├── layernorm_deepspeed.gif │ │ │ ├── layernorm_ds.png │ │ │ ├── layernorm_pytorch.gif │ │ │ ├── layernorm_torch.png │ │ │ ├── lightning-dark.png │ │ │ ├── lightning-dark.svg │ │ │ ├── lightning-light.svg │ │ │ ├── lightning.png │ │ │ ├── loss_and_lr.png │ │ │ ├── lr_schedule.png │ │ │ ├── megatron-gpt2-perf-test.png │ │ │ ├── mii │ │ │ ├── azure-cost.png │ │ │ ├── bert.png │ │ │ ├── bloom.png │ │ │ ├── gpt.png │ │ │ ├── hero-transparent.png │ │ │ ├── hero.png │ │ │ ├── llm-latency-sd-latency-zoom.png │ │ │ ├── llm-latency-sd-latency.png │ │ │ ├── mii-arch.png │ │ │ ├── multi-gpu-latency.png │ │ │ ├── opt-bloom.png │ │ │ ├── opt.png │ │ │ ├── roberta.png │ │ │ ├── sd-latency.png │ │ │ └── tput-llms.png │ │ │ ├── model_convergence.png │ │ │ ├── moe-lat-tput.png │ │ │ ├── moe-nlg.png │ │ │ ├── mosaicml.svg │ │ │ ├── old-vs-new-azure.png │ │ │ ├── onebit-adam-overview.png │ │ │ ├── onebit-convergence.png │ │ │ ├── oom_dp8_1.5B_log.png │ │ │ ├── perf-overview.png │ │ │ ├── pipe-schedule.png │ │ │ ├── pp-lowbw-gpt2.png │ │ │ ├── prmoe.png │ │ │ ├── qkv_fusion.png │ │ │ ├── quantization-8bit.png │ │ │ ├── quantization-mixedbit.png │ │ │ ├── sa_backward_pass.png │ │ │ ├── sa_bert_base_time_result.png │ │ │ ├── sa_bert_large_time_result.png │ │ │ ├── sa_fixed_sparsity_structure.png │ │ │ ├── sa_forward_pass.png │ │ │ ├── sa_gpt2_time_result.png │ │ │ ├── sa_long_document_comprehension_result.png │ │ │ ├── sa_maximum_sequence_runnable_on_bert.png │ │ │ ├── sa_variable_sparsity_structure.png │ │ │ ├── softmax_animation.gif │ │ │ ├── softmax_deepspeed.gif │ │ │ ├── softmax_ds.png │ │ │ ├── softmax_pytorch.gif │ │ │ ├── softmax_torch.png │ │ │ ├── squad-ib.png │ │ │ ├── squad-scaling.png │ │ │ ├── squad-tcp.png │ │ │ ├── tensorboard_monitor.PNG │ │ │ ├── transformer_kernel_perf.png │ │ │ ├── transformer_kernel_perf_seq128.PNG │ │ │ ├── transformer_kernel_perf_seq512.PNG │ │ │ ├── transformer_preln_arch.png │ │ │ ├── transformers-dark.png │ │ │ ├── transformers-light.png │ │ │ ├── variable_sparsity_pattern.png │ │ │ ├── vmss-setup.png │ │ │ ├── wandb_monitor.PNG │ │ │ ├── webinar-aug2020.png │ │ │ ├── xtc-1.png │ │ │ ├── xtc-2.png │ │ │ ├── xtc-3.png │ │ │ ├── xtc-4.png │ │ │ ├── zero-full.png │ │ │ ├── zero1_dp8_1.5B_log.png │ │ │ ├── zero1_dp8_1.5B_smi.png │ │ │ ├── zero2_dp32_10B_log.png │ │ │ ├── zero2_dp32_10B_smi.png │ │ │ ├── zero3-offload-1-v100.png │ │ │ ├── zero3-offload-16-v100.png │ │ │ ├── zero3-offload-200B-scalability.png │ │ │ ├── zero3-offload-512-v100.png │ │ │ ├── zero3-offload-memory-overview.png │ │ │ ├── zero_inference_full_offload.png │ │ │ ├── zero_inference_model_scale.png │ │ │ ├── zero_inference_models.png │ │ │ ├── zero_inference_multi_gpu.png │ │ │ ├── zero_inference_prefetch.png │ │ │ ├── zero_inference_token_count_batch_size.png │ │ │ ├── zero_inference_token_count_cpu_throughput.png │ │ │ ├── zero_inference_token_count_nvme_throughput.png │ │ │ ├── zero_offload_dp1_10B_cpu.png │ │ │ ├── zero_offload_dp1_10B_log.png │ │ │ └── zero_offload_dp1_10B_smi.png │ ├── code-docs │ │ ├── Makefile │ │ ├── build-api-docs.sh │ │ └── source │ │ │ ├── activation-checkpointing.rst │ │ │ ├── autotuning.rst │ │ │ ├── conf.py │ │ │ ├── flops-profiler.rst │ │ │ ├── index.rst │ │ │ ├── inference-engine.rst │ │ │ ├── inference-init.rst │ │ │ ├── initialize.rst │ │ │ ├── kernel.rst │ │ │ ├── memory.rst │ │ │ ├── model-checkpointing.rst │ │ │ ├── moe.rst │ │ │ ├── optimizers.rst │ │ │ ├── pipeline.rst │ │ │ ├── schedulers.rst │ │ │ ├── training.rst │ │ │ └── zero3.rst │ ├── contributing.md │ └── index.md ├── examples │ └── README.md ├── install.sh ├── op_builder │ ├── __init__.py │ ├── async_io.py │ ├── builder.py │ ├── cpu_adagrad.py │ ├── cpu_adam.py │ ├── fused_adam.py │ ├── fused_lamb.py │ ├── quantizer.py │ ├── sparse_attn.py │ ├── stochastic_transformer.py │ ├── transformer.py │ ├── transformer_inference.py │ └── utils.py ├── release │ ├── bump_patch_version.py │ └── release.sh ├── requirements │ ├── requirements-1bit-mpi.txt │ ├── requirements-autotuning-ml.txt │ ├── requirements-autotuning.txt │ ├── requirements-dev.txt │ ├── requirements-inf.txt │ ├── requirements-readthedocs.txt │ ├── requirements-sd.txt │ ├── requirements-sparse_attn.txt │ └── requirements.txt ├── scripts │ └── check-torchdist.py ├── setup.cfg ├── setup.py ├── tests │ ├── benchmarks │ │ ├── flatten_bench.py │ │ └── unflatten_bench.py │ ├── conftest.py │ ├── lightning │ │ └── test_simple.py │ ├── model │ │ ├── BingBertSquad │ │ │ ├── BingBertSquad_run_func_test.py │ │ │ ├── BingBertSquad_test_common.py │ │ │ ├── __init__.py │ │ │ ├── deepspeed_bsz24_fp16_config.json │ │ │ ├── deepspeed_bsz24_fp16_eigenvalue_quantize_config.json │ │ │ ├── deepspeed_bsz24_fp16_zero2_config.json │ │ │ ├── deepspeed_bsz24_fp32_config.json │ │ │ ├── run_BingBertSquad.sh │ │ │ ├── run_BingBertSquad_sanity.sh │ │ │ ├── run_tests.sh │ │ │ └── test_e2e_squad.py │ │ ├── Megatron_GPT2 │ │ │ ├── __init__.py │ │ │ ├── ds_config_func_bs4_zero1.json │ │ │ ├── ds_config_func_bs4_zero2.json │ │ │ ├── ds_config_func_bs4_zero2_offload.json │ │ │ ├── ds_config_func_bs8_no_zero.json │ │ │ ├── ds_config_func_bs8_zero0_gas3.json │ │ │ ├── ds_config_func_bs8_zero1.json │ │ │ ├── ds_config_func_bs8_zero2.json │ │ │ ├── ds_config_func_bs8_zero2_gas3.json │ │ │ ├── ds_config_func_bs8_zero2_offload.json │ │ │ ├── ds_config_func_scheduler.json │ │ │ ├── ds_config_perf_bs16.json │ │ │ ├── ds_config_perf_bs32.json │ │ │ ├── ds_config_perf_bs8.json │ │ │ ├── ds_gpt2_test.sh │ │ │ ├── run_checkpoint_test.py │ │ │ ├── run_func_test.py │ │ │ ├── run_perf_baseline.py │ │ │ ├── run_perf_test.py │ │ │ └── test_common.py │ │ └── run_sanity_check.py │ ├── onebit │ │ ├── test_mpi_backend.py │ │ ├── test_mpi_perf.py │ │ ├── test_nccl_backend.py │ │ └── test_nccl_perf.py │ ├── perf │ │ ├── adam_test.py │ │ └── adam_test1.py │ ├── pytest.ini │ ├── small_model_debugging │ │ ├── stage3_test.py │ │ ├── test.py │ │ └── test_model.py │ └── unit │ │ ├── __init__.py │ │ ├── alexnet_model.py │ │ ├── autotuning │ │ └── test_autotuning.py │ │ ├── checkpoint │ │ ├── common.py │ │ ├── test_latest_checkpoint.py │ │ ├── test_lr_scheduler.py │ │ ├── test_moe_checkpoint.py │ │ ├── test_other_optimizer.py │ │ ├── test_pipeline.py │ │ ├── test_reshape_checkpoint.py │ │ ├── test_sparse.py │ │ ├── test_tag_validation.py │ │ └── test_zero_optimizer.py │ │ ├── comm │ │ └── test_dist.py │ │ ├── common.py │ │ ├── compression │ │ └── test_compression.py │ │ ├── ds_batch_config.json │ │ ├── elasticity │ │ └── test_elastic.py │ │ ├── gpt2-merges.txt │ │ ├── gpt2-vocab.json │ │ ├── inference │ │ └── test_inference.py │ │ ├── launcher │ │ ├── test_ds_arguments.py │ │ ├── test_multinode_runner.py │ │ └── test_run.py │ │ ├── megatron_model.py │ │ ├── model_parallelism │ │ ├── test_configurable_parallel_mp.py │ │ └── test_configurable_parallel_pp.py │ │ ├── modeling.py │ │ ├── modelingpreln.py │ │ ├── moe │ │ ├── test_moe.py │ │ └── test_moe_tp.py │ │ ├── monitor │ │ └── test_monitor.py │ │ ├── multi_output_model.py │ │ ├── ops │ │ ├── adagrad │ │ │ └── test_cpu_adagrad.py │ │ ├── adam │ │ │ ├── test_adamw.py │ │ │ └── test_cpu_adam.py │ │ ├── aio │ │ │ └── test_aio.py │ │ ├── cuda │ │ │ ├── test_cuda_backward.py │ │ │ └── test_cuda_forward.py │ │ ├── quantizer │ │ │ └── test_quant.py │ │ ├── sparse_attention │ │ │ └── test_sparse_attention.py │ │ └── transformer │ │ │ └── inference │ │ │ ├── test_bias_add.py │ │ │ ├── test_bias_gelu.py │ │ │ ├── test_bias_relu.py │ │ │ ├── test_moe_res_matmult.py │ │ │ └── test_residual_add.py │ │ ├── pipe │ │ └── test_pipe_module.py │ │ ├── profiling │ │ └── flops_profiler │ │ │ └── test_flops_profiler.py │ │ ├── runtime │ │ ├── activation_checkpointing │ │ │ └── test_activation_checkpointing.py │ │ ├── comm │ │ │ └── test_coalesced_collectives.py │ │ ├── half_precision │ │ │ ├── onebit │ │ │ │ └── test_onebit.py │ │ │ ├── test_bf16.py │ │ │ ├── test_dynamic_loss_scale.py │ │ │ └── test_fp16.py │ │ ├── pipe │ │ │ ├── test_pipe.py │ │ │ ├── test_pipe_schedule.py │ │ │ └── test_topology.py │ │ ├── sparse_tensor │ │ │ ├── test_averaging_sparse_gradients.py │ │ │ ├── test_csr.py │ │ │ └── test_sparse_grads.py │ │ ├── test_autocast.py │ │ ├── test_curriculum_learning.py │ │ ├── test_data.py │ │ ├── test_ds_config_dict.py │ │ ├── test_ds_config_model.py │ │ ├── test_ds_initialize.py │ │ ├── test_lr_schedulers.py │ │ ├── test_multi_output_model.py │ │ ├── test_pld.py │ │ ├── test_runtime_utils.py │ │ ├── utils │ │ │ └── test_partition.py │ │ └── zero │ │ │ ├── test_ignore_unused_parameters.py │ │ │ ├── test_zero.py │ │ │ ├── test_zero_config.py │ │ │ ├── test_zero_context.py │ │ │ └── test_zero_tiled.py │ │ ├── simple_model.py │ │ ├── util.py │ │ └── utils │ │ ├── test_get_optim_files.py │ │ ├── test_groups.py │ │ └── test_init_on_device.py └── version.txt ├── Megatron-DeepSpeed ├── .gitignore ├── CODEOWNERS ├── LICENSE ├── MANIFEST.in ├── README.md ├── SECURITY.md ├── dataset │ ├── README.md │ ├── download_books.sh │ ├── download_ckpt.sh │ └── download_vocab.sh ├── examples │ ├── EnvPipe │ │ ├── config │ │ │ └── TEMPLATE.json │ │ ├── run_megatron_1.3B_distributed.sh │ │ ├── run_megatron_125M.sh │ │ ├── run_megatron_350M.sh │ │ ├── run_megatron_760M.sh │ │ └── run_megatron_760M_distributed.sh │ ├── MoE │ │ ├── ds_config_gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus--1-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-27638-token-60B.json │ │ ├── ds_config_gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus--2-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-27638-token-60B.json │ │ ├── ds_config_gpt3-with-pile-0.125B-lr-2.4e-3-minlr-6.0e-5-bs-2048-gpus-4-zero-0-mp-1-pp-1-no_pp-cl-startseqlen-72-step-27638-token-60B.json │ │ ├── ds_config_gpt_TEMPLATE.json │ │ ├── ds_config_gpt_Zero2_TEMPLATE.json │ │ ├── ds_config_gpt_gpt-0.125B-lr-4.5e-4-minlr-4.5e-06-bs-256-gpus--2-mp-1-pp-1-ep-4-mlc-0.01-cap-1.0-drop-false.json │ │ ├── ds_config_gpt_gpt-0.125B-lr-4.5e-4-minlr-4.5e-06-bs-256-gpus--2-mp-1-pp-1-ep-64-mlc-0.01-cap-1.0-drop-true.json │ │ ├── ds_config_gpt_gpt-0.125B-lr-4.5e-4-minlr-4.5e-06-bs-256-gpus-4-mp-1-pp-1-ep-2-mlc-0.01-cap-1.0-drop-true.json │ │ ├── ds_config_gpt_gpt-0.125B-lr-4.5e-4-minlr-4.5e-06-bs-256-gpus-4-mp-1-pp-1-ep-3-mlc-0.01-cap-1.0-drop-true.json │ │ ├── ds_config_gpt_gpt-0.125B-lr-4.5e-4-minlr-4.5e-06-bs-256-gpus-4-mp-1-pp-1-ep-4-mlc-0.01-cap-1.0-drop-false.json │ │ ├── ds_config_gpt_gpt-0.125B-lr-4.5e-4-minlr-4.5e-06-bs-256-gpus-4-mp-1-pp-1-ep-4-mlc-0.01-cap-1.0-drop-true.json │ │ ├── ds_config_gpt_gpt-0.125B-lr-4.5e-4-minlr-4.5e-06-bs-256-gpus-4-mp-1-pp-1-ep-64-mlc-0.01-cap-1.0-drop-false.json │ │ ├── ds_config_gpt_gpt-0.125B-lr-4.5e-4-minlr-4.5e-06-bs-256-gpus-4-mp-1-pp-1-ep-64-mlc-0.01-cap-1.0-drop-true.json │ │ ├── ds_config_gpt_gpt-0.125B-lr-4.5e-4-minlr-4.5e-06-bs-256-gpus-4-mp-1-pp-1-ep-64-mlc-0.01-cap-1.5-drop-true.json │ │ ├── ds_config_gpt_gpt-0.125B-lr-4.5e-4-minlr-4.5e-06-bs-256-gpus-8-mp-1-pp-1-ep-64-mlc-0.01-cap-1.0-drop-true.json │ │ ├── ds_config_gpt_gpt-0.35B-lr-3.0e-4-minlr-3.0e-5-bs-256-gpus-64-mp-1-pp-1.json │ │ ├── ds_evalharness.sh │ │ ├── ds_pretrain_gpt_1.3B_MoE128.sh │ │ ├── ds_pretrain_gpt_1.3B_PR-MoE64or128.sh │ │ ├── ds_pretrain_gpt_1.3B_PR-MoE64or128_MoS.sh │ │ ├── ds_pretrain_gpt_1.3B_dense.sh │ │ ├── ds_pretrain_gpt_1.3B_dense_cl.sh │ │ ├── ds_pretrain_gpt_125M_MoE64.sh │ │ ├── ds_pretrain_gpt_125M_dense_cl.sh │ │ ├── ds_pretrain_gpt_350M_MoE128.sh │ │ ├── ds_pretrain_gpt_350M_PR-MoE32or64.sh │ │ ├── ds_pretrain_gpt_350M_PR-MoE32or64_MoS.sh │ │ ├── ds_pretrain_gpt_350M_dense.sh │ │ ├── ds_pretrain_gpt_6.7B_dense.sh │ │ ├── profile.py │ │ └── readme_evalharness.md │ ├── README.md │ ├── azure │ │ ├── README.md │ │ ├── run-175b.sh │ │ ├── run-1t.sh │ │ └── run-benchmark-model.sh │ ├── azureml │ │ ├── Dockerfile.dockerfile │ │ ├── README.md │ │ ├── aml_submit.py │ │ └── prepare_dataset.py │ ├── bert_with_pile │ │ ├── README.md │ │ ├── bert-large-uncased-vocab.txt │ │ ├── ds_config_bert_TEMPLATE.json │ │ ├── ds_config_bert_bsz1024_mbsz-1024_log100_zero0.json │ │ ├── ds_config_bert_bsz1024_mbsz256_log100_zero0.json │ │ ├── ds_config_bert_bsz1024_mbsz_log100_zero0.json │ │ ├── ds_config_bert_bsz128_mbsz32_log100_zero0.json │ │ ├── ds_config_bert_bsz256_mbsz64_log100_zero0.json │ │ ├── ds_config_bert_bsz32_mbsz8_log100_zero0.json │ │ ├── ds_config_bert_bsz512_mbsz128_log100_zero0.json │ │ ├── ds_config_bert_bsz64_mbsz16_log100_zero0.json │ │ ├── ds_config_bert_bsz64_mbsz64_log100_zero0.json │ │ ├── ds_config_bert_bsz64_mbsz8_log100_zero0.json │ │ ├── ds_finetune_bert_mnli.sh │ │ ├── ds_finetune_bert_qqp.sh │ │ ├── ds_finetune_bert_race.sh │ │ ├── ds_pretrain_bert.sh │ │ └── prepare_pile_data.py │ ├── compression │ │ ├── 125M-Int8-test-64gpu-distilled-group48.sh │ │ ├── 125M-L10-Int8-test-64gpu-distilled-group48.sh │ │ ├── 125M-L12-Int8-test-64gpu-distilled-group48.sh │ │ ├── ds_config_gpt_TEMPLATE.json │ │ ├── ds_config_gpt_TEMPLATE_compression.json │ │ ├── ds_evalharness.sh │ │ ├── ds_pretrain_gpt_1.3B_dense_cl_kd.sh │ │ ├── ds_pretrain_gpt_125M_dense_cl_kd.sh │ │ ├── ds_pretrain_gpt_125M_dense_kd.sh │ │ └── ds_pretrain_gpt_350M_dense_kd.sh │ ├── create_embeddings.sh │ ├── curriculum_learning │ │ ├── README.md │ │ ├── ds_pretrain_gpt2.sh │ │ ├── ds_train.sh │ │ ├── ds_zero_stage_1_config_baseline.json │ │ └── ds_zero_stage_1_config_curriculum_fixed_linear.json │ ├── evaluate_ict_zeroshot_nq.sh │ ├── evaluate_zeroshot_gpt.sh │ ├── finetune_mnli_distributed.sh │ ├── finetune_race_distributed.sh │ ├── generate_text.sh │ ├── merge_mp_bert.sh │ ├── pretrain_bert.sh │ ├── pretrain_bert_distributed.sh │ ├── pretrain_bert_distributed_with_mp.sh │ ├── pretrain_gpt.sh │ ├── pretrain_gpt3_175B.sh │ ├── pretrain_gpt_distributed.sh │ ├── pretrain_gpt_distributed_with_mp.sh │ ├── pretrain_ict.sh │ ├── pretrain_t5.sh │ ├── pretrain_t5_distributed.sh │ ├── pretrain_t5_distributed_with_mp.sh │ └── run_deepspeed_example.sh ├── images │ └── cases_april2021.png ├── megatron │ ├── __init__.py │ ├── arguments.py │ ├── checkpointing.py │ ├── data │ │ ├── Makefile │ │ ├── __init__.py │ │ ├── autoaugment.py │ │ ├── bert_dataset.py │ │ ├── biencoder_dataset_utils.py │ │ ├── blendable_dataset.py │ │ ├── data_samplers.py │ │ ├── dataset_utils.py │ │ ├── gpt_dataset.py │ │ ├── helpers.cpp │ │ ├── ict_dataset.py │ │ ├── indexed_dataset.py │ │ ├── orqa_wiki_dataset.py │ │ ├── realm_dataset_utils.py │ │ ├── realm_index.py │ │ ├── t5_dataset.py │ │ ├── test │ │ │ ├── test_indexed_dataset.py │ │ │ └── test_preprocess_data.sh │ │ └── vit_dataset.py │ ├── enums.py │ ├── fp16_deprecated │ │ └── loss_scaler.py │ ├── fused_kernels │ │ ├── __init__.py │ │ ├── compat.h │ │ ├── layer_norm_cuda.cpp │ │ ├── layer_norm_cuda_kernel.cu │ │ ├── scaled_masked_softmax.cpp │ │ ├── scaled_masked_softmax.h │ │ ├── scaled_masked_softmax_cuda.cu │ │ ├── scaled_upper_triang_masked_softmax.cpp │ │ ├── scaled_upper_triang_masked_softmax.h │ │ ├── scaled_upper_triang_masked_softmax_cuda.cu │ │ └── type_shim.h │ ├── global_vars.py │ ├── indexer.py │ ├── initialize.py │ ├── learning_rates.py │ ├── memory.py │ ├── microbatches.py │ ├── model │ │ ├── __init__.py │ │ ├── bert_model.py │ │ ├── biencoder_model.py │ │ ├── classification.py │ │ ├── distributed.py │ │ ├── enums.py │ │ ├── fused_bias_gelu.py │ │ ├── fused_layer_norm.py │ │ ├── fused_softmax.py │ │ ├── gpt_model.py │ │ ├── language_model.py │ │ ├── module.py │ │ ├── multiple_choice.py │ │ ├── realm_model.py │ │ ├── t5_model.py │ │ ├── transformer.py │ │ ├── utils.py │ │ └── vit_model.py │ ├── mpu │ │ ├── __init__.py │ │ ├── cross_entropy.py │ │ ├── data.py │ │ ├── initialize.py │ │ ├── layers.py │ │ ├── mappings.py │ │ ├── random.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── commons.py │ │ │ ├── test_cross_entropy.py │ │ │ ├── test_data.py │ │ │ ├── test_initialize.py │ │ │ ├── test_layers.py │ │ │ └── test_random.py │ │ └── utils.py │ ├── optimizer │ │ ├── __init__.py │ │ ├── clip_grads.py │ │ ├── grad_scaler.py │ │ └── optimizer.py │ ├── p2p_communication.py │ ├── package_info.py │ ├── schedules.py │ ├── text_generation_utils.py │ ├── tokenizer │ │ ├── __init__.py │ │ ├── bert_tokenization.py │ │ ├── gpt2_tokenization.py │ │ └── tokenizer.py │ ├── training.py │ └── utils.py ├── pretrain_bert.py ├── pretrain_gpt.py ├── pretrain_ict.py ├── pretrain_t5.py ├── pretrain_vit.py ├── requirements.txt ├── setup.py ├── tasks │ ├── data_utils.py │ ├── ensemble_classifier.py │ ├── eval_harness │ │ ├── download.py │ │ ├── evaluate.py │ │ └── report-to-csv.py │ ├── eval_utils.py │ ├── finetune_utils.py │ ├── glue │ │ ├── data.py │ │ ├── finetune.py │ │ ├── mnli.py │ │ └── qqp.py │ ├── main.py │ ├── orqa │ │ ├── evaluate_orqa.py │ │ ├── evaluate_utils.py │ │ └── natural_questions │ │ │ ├── nq.py │ │ │ ├── qa_utils.py │ │ │ └── tokenizers.py │ ├── race │ │ ├── data.py │ │ └── finetune.py │ ├── vision │ │ ├── classification.py │ │ ├── eval_utils.py │ │ ├── finetune_utils.py │ │ └── main.py │ └── zeroshot_gpt │ │ ├── datasets.py │ │ ├── detokenizer.py │ │ └── evaluate.py ├── tests │ └── test_basic.py └── tools │ ├── convert_checkpoint │ ├── README.md │ ├── deepspeed_checkpoint.py │ ├── deepspeed_to_megatron.py │ ├── deepspeed_to_transformers.py │ ├── inspect_checkpoint.py │ └── inspect_deepspeed_checkpoint.py │ ├── create_doc_index.py │ ├── generate_samples_gpt.py │ ├── linter.py │ ├── merge_mp_partitions.py │ ├── openwebtext │ ├── README.md │ ├── add_id.py │ ├── blacklist_urls.py │ ├── cleanup_dataset.py │ ├── cleanup_fix_dataset.py │ ├── filter_ngrams.py │ ├── find_duplicates.py │ ├── group_duplicate_url.py │ ├── merge_jsons.py │ └── remove_group_duplicates.py │ └── preprocess_data.py ├── README.md └── benchmark ├── multi_node ├── model │ └── gpt.py └── script │ ├── config │ └── TEMPLATE.json │ ├── fig12.sh │ ├── run_gpt_350M.sh │ ├── run_megatron_1.3B.sh │ └── run_megatron_760M.sh └── single_node ├── model ├── bert.py ├── gpt.py └── resnet.py └── script ├── config └── TEMPLATE.json ├── fig10.sh ├── fig11.sh ├── fig13.sh ├── fig9_a.sh ├── fig9_b.sh ├── run_bert_1.3B.sh ├── run_bert_3.9B.sh ├── run_bert_336M.sh ├── run_gpt_125M.sh ├── run_gpt_350M.sh ├── run_megatron_125M.sh ├── run_megatron_350M.sh ├── run_megatron_760M.sh ├── run_resnet_152.sh ├── single_3090.sh └── single_v100.sh /.gitignore: -------------------------------------------------------------------------------- 1 | env/ 2 | tmp.out 3 | *.csv 4 | *.json 5 | -------------------------------------------------------------------------------- /DeepSpeed/.clang-format: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/.clang-format -------------------------------------------------------------------------------- /DeepSpeed/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/.gitignore -------------------------------------------------------------------------------- /DeepSpeed/.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/.pre-commit-config.yaml -------------------------------------------------------------------------------- /DeepSpeed/.pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/.pylintrc -------------------------------------------------------------------------------- /DeepSpeed/.readthedocs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/.readthedocs.yml -------------------------------------------------------------------------------- /DeepSpeed/.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | SPLIT_ALL_COMMA_SEPARATED_VALUES = true 3 | COLUMN_LIMIT = 89 4 | -------------------------------------------------------------------------------- /DeepSpeed/CODEOWNERS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/CODEOWNERS -------------------------------------------------------------------------------- /DeepSpeed/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /DeepSpeed/CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/CONTRIBUTING.md -------------------------------------------------------------------------------- /DeepSpeed/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/LICENSE -------------------------------------------------------------------------------- /DeepSpeed/MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/MANIFEST.in -------------------------------------------------------------------------------- /DeepSpeed/MANIFEST_win.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/MANIFEST_win.in -------------------------------------------------------------------------------- /DeepSpeed/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/README.md -------------------------------------------------------------------------------- /DeepSpeed/SECURITY.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/SECURITY.md -------------------------------------------------------------------------------- /DeepSpeed/azure/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/azure/README.md -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/communication/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/communication/README.md -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/communication/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/communication/all_gather.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/communication/all_gather.py -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/communication/all_reduce.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/communication/all_reduce.py -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/communication/all_to_all.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/communication/all_to_all.py -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/communication/broadcast.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/communication/broadcast.py -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/communication/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/communication/constants.py -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/communication/pt2pt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/communication/pt2pt.py -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/communication/run_all.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/communication/run_all.py -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/communication/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/communication/utils.py -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/inference/bert-bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/inference/bert-bench.py -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/inference/collect_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/inference/collect_results.py -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/inference/gpt-bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/inference/gpt-bench.py -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/inference/requirements.txt: -------------------------------------------------------------------------------- 1 | transformers>=4.21.3 2 | -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/inference/run_model.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/inference/run_model.sh -------------------------------------------------------------------------------- /DeepSpeed/benchmarks/inference/sweep.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/benchmarks/inference/sweep.sh -------------------------------------------------------------------------------- /DeepSpeed/bin/deepspeed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/bin/deepspeed -------------------------------------------------------------------------------- /DeepSpeed/bin/deepspeed.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/bin/deepspeed.pt -------------------------------------------------------------------------------- /DeepSpeed/bin/ds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/bin/ds -------------------------------------------------------------------------------- /DeepSpeed/bin/ds_bench: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/bin/ds_bench -------------------------------------------------------------------------------- /DeepSpeed/bin/ds_elastic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/bin/ds_elastic -------------------------------------------------------------------------------- /DeepSpeed/bin/ds_report: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/bin/ds_report -------------------------------------------------------------------------------- /DeepSpeed/bin/ds_ssh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/bin/ds_ssh -------------------------------------------------------------------------------- /DeepSpeed/bin/dsr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/bin/dsr -------------------------------------------------------------------------------- /DeepSpeed/csrc/adagrad/cpu_adagrad.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/adagrad/cpu_adagrad.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/adam/cpu_adam.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/adam/cpu_adam.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/adam/fused_adam_frontend.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/adam/fused_adam_frontend.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/adam/multi_tensor_adam.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/adam/multi_tensor_adam.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/adam/multi_tensor_apply.cuh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/adam/multi_tensor_apply.cuh -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/common/deepspeed_aio_common.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/common/deepspeed_aio_common.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/common/deepspeed_aio_common.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/common/deepspeed_aio_common.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/common/deepspeed_aio_types.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/common/deepspeed_aio_types.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/common/deepspeed_aio_types.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/common/deepspeed_aio_types.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/common/deepspeed_aio_utils.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/common/deepspeed_aio_utils.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/common/deepspeed_aio_utils.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/common/deepspeed_aio_utils.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_lib/deepspeed_aio_thread.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_lib/deepspeed_aio_thread.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_lib/deepspeed_aio_thread.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_lib/deepspeed_aio_thread.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_lib/deepspeed_py_aio.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_lib/deepspeed_py_aio.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_lib/deepspeed_py_aio.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_lib/deepspeed_py_aio.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_lib/deepspeed_py_aio_handle.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_lib/deepspeed_py_aio_handle.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_lib/deepspeed_py_aio_handle.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_lib/deepspeed_py_aio_handle.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_lib/deepspeed_py_copy.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_lib/deepspeed_py_copy.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_lib/deepspeed_py_copy.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_lib/deepspeed_py_copy.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_lib/py_ds_aio.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_lib/py_ds_aio.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/aio_bench_generate_param.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/aio_bench_generate_param.py -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/aio_bench_perf_sweep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/aio_bench_perf_sweep.py -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/ds_aio_basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/ds_aio_basic.py -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/ds_aio_handle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/ds_aio_handle.py -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/parse_aio_stats.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/parse_aio_stats.py -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/perf_sweep_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/perf_sweep_utils.py -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/run_read_sweep.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/run_read_sweep.sh -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/run_write_sweep.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/run_write_sweep.sh -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/single_process_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/single_process_config.json -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/test_ds_aio.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/test_ds_aio.py -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/test_ds_aio_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/test_ds_aio_utils.py -------------------------------------------------------------------------------- /DeepSpeed/csrc/aio/py_test/validate_async_io.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/aio/py_test/validate_async_io.py -------------------------------------------------------------------------------- /DeepSpeed/csrc/common/custom_cuda_kernel.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/common/custom_cuda_kernel.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/StopWatch.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/StopWatch.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/Timer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/Timer.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/compat.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/compat.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/context.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/context.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/conversion_utils.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/conversion_utils.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/cpu_adagrad.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/cpu_adagrad.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/cpu_adam.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/cpu_adam.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/cublas_wrappers.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/cublas_wrappers.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/custom_cuda_layers.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/custom_cuda_layers.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/dropout.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/dropout.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/ds_kernel_utils.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/ds_kernel_utils.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/ds_transformer_cuda.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/ds_transformer_cuda.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/feed_forward.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/feed_forward.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/gelu.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/gelu.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/gemm_test.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/gemm_test.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/general_kernels.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/general_kernels.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/memory_access_utils.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/memory_access_utils.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/normalize_layer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/normalize_layer.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/quantizer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/quantizer.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/simd.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/simd.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/softmax.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/softmax.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/strided_batch_gemm.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/strided_batch_gemm.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/includes/type_shim.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/includes/type_shim.h -------------------------------------------------------------------------------- /DeepSpeed/csrc/lamb/fused_lamb_cuda.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/lamb/fused_lamb_cuda.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/lamb/fused_lamb_cuda_kernel.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/lamb/fused_lamb_cuda_kernel.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/quantization/pt_binding.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/quantization/pt_binding.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/quantization/quantizer.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/quantization/quantizer.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/sparse_attention/utils.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/sparse_attention/utils.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/cublas_wrappers.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/cublas_wrappers.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/dropout_kernels.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/dropout_kernels.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/ds_transformer_cuda.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/ds_transformer_cuda.cpp -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/gelu_kernels.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/gelu_kernels.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/general_kernels.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/general_kernels.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/inference/csrc/gelu.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/inference/csrc/gelu.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/inference/csrc/normalize.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/inference/csrc/normalize.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/inference/csrc/relu.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/inference/csrc/relu.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/inference/csrc/softmax.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/inference/csrc/softmax.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/inference/csrc/transform.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/inference/csrc/transform.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/normalize_kernels.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/normalize_kernels.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/softmax_kernels.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/softmax_kernels.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/transformer/transform_kernels.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/transformer/transform_kernels.cu -------------------------------------------------------------------------------- /DeepSpeed/csrc/utils/flatten_unflatten.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/csrc/utils/flatten_unflatten.cpp -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/.gitignore -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/README.md -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/autotuner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/autotuner.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/config.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/scheduler.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/tuner/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/tuner/README.md -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/tuner/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/tuner/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/tuner/base_tuner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/tuner/base_tuner.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/tuner/cost_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/tuner/cost_model.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/tuner/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/tuner/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/autotuning/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/autotuning/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/checkpoint/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/checkpoint/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/checkpoint/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/checkpoint/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/checkpoint/deepspeed_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/checkpoint/deepspeed_checkpoint.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/checkpoint/reshape_3d_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/checkpoint/reshape_3d_utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/checkpoint/reshape_meg_2d.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/checkpoint/reshape_meg_2d.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/checkpoint/reshape_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/checkpoint/reshape_utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/checkpoint/universal_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/checkpoint/universal_checkpoint.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/checkpoint/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/checkpoint/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/checkpoint/zero_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/checkpoint/zero_checkpoint.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/comm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/comm/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/comm/backend.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/comm/backend.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/comm/comm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/comm/comm.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/comm/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/comm/config.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/comm/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/comm/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/comm/torch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/comm/torch.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/comm/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/comm/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/compression/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/compression/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/compression/basic_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/compression/basic_layer.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/compression/compress.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/compression/compress.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/compression/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/compression/config.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/compression/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/compression/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/compression/helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/compression/helper.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/compression/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/compression/scheduler.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/compression/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/compression/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/elasticity/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/elasticity/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/elasticity/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/elasticity/config.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/elasticity/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/elasticity/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/elasticity/elastic_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/elasticity/elastic_agent.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/elasticity/elasticity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/elasticity/elasticity.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/elasticity/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/elasticity/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/env_report.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/env_report.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/git_version_info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/git_version_info.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/inference/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/inference/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/inference/engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/inference/engine.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/launcher/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/launcher/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/launcher/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/launcher/launch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/launcher/launch.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/launcher/multinode_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/launcher/multinode_runner.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/launcher/runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/launcher/runner.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/module_inject/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/module_inject/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/module_inject/encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/module_inject/encoder.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/module_inject/inject.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/module_inject/inject.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/module_inject/layers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/module_inject/layers.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/module_inject/load_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/module_inject/load_checkpoint.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/module_inject/module_quantize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/module_inject/module_quantize.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/module_inject/replace_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/module_inject/replace_module.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/module_inject/replace_policy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/module_inject/replace_policy.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/module_inject/unet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/module_inject/unet.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/moe/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/moe/experts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/moe/experts.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/moe/layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/moe/layer.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/moe/mappings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/moe/mappings.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/moe/sharded_moe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/moe/sharded_moe.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/moe/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/moe/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/monitor/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/monitor/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/monitor/config.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/monitor/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/monitor/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/monitor/csv_monitor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/monitor/csv_monitor.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/monitor/monitor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/monitor/monitor.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/monitor/tensorboard.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/monitor/tensorboard.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/monitor/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/monitor/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/monitor/wandb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/monitor/wandb.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/nebula/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/nebula/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/nebula/config.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/nebula/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/nebula/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/adagrad/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/adagrad/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/adagrad/cpu_adagrad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/adagrad/cpu_adagrad.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/adam/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/adam/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/adam/cpu_adam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/adam/cpu_adam.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/adam/fused_adam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/adam/fused_adam.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/adam/multi_tensor_apply.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/adam/multi_tensor_apply.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/aio/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/aio/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/adagrad/cpu_adagrad.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/adagrad/cpu_adagrad.cpp -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/adam/cpu_adam.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/adam/cpu_adam.cpp -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/adam/multi_tensor_adam.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/adam/multi_tensor_adam.cu -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/aio/py_lib/py_ds_aio.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/aio/py_lib/py_ds_aio.cpp -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/StopWatch.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/StopWatch.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/Timer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/Timer.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/compat.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/compat.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/context.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/context.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/cpu_adagrad.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/cpu_adagrad.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/cpu_adam.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/cpu_adam.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/dropout.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/dropout.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/feed_forward.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/feed_forward.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/gelu.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/gelu.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/gemm_test.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/gemm_test.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/quantizer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/quantizer.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/simd.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/simd.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/softmax.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/softmax.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/includes/type_shim.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/includes/type_shim.h -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/lamb/fused_lamb_cuda.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/lamb/fused_lamb_cuda.cpp -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/csrc/quantization/quantizer.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/csrc/quantization/quantizer.cu -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/lamb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/lamb/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/lamb/fused_lamb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/lamb/fused_lamb.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/op_builder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/op_builder/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/op_builder/async_io.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/op_builder/async_io.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/op_builder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/op_builder/builder.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/op_builder/cpu_adagrad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/op_builder/cpu_adagrad.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/op_builder/cpu_adam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/op_builder/cpu_adam.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/op_builder/fused_adam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/op_builder/fused_adam.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/op_builder/fused_lamb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/op_builder/fused_lamb.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/op_builder/quantizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/op_builder/quantizer.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/op_builder/sparse_attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/op_builder/sparse_attn.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/op_builder/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/op_builder/transformer.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/op_builder/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/op_builder/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/quantizer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/quantizer/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/quantizer/quantizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/quantizer/quantizer.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/sparse_attention/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/sparse_attention/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/sparse_attention/matmul.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/sparse_attention/matmul.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/sparse_attention/softmax.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/sparse_attention/softmax.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/transformer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/transformer/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/ops/transformer/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/ops/transformer/transformer.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/pipe/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/pipe/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/profiling/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/profiling/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/profiling/config.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/profiling/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/profiling/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/profiling/energy_profiler/__init__.py: -------------------------------------------------------------------------------- 1 | from .profiler import * 2 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/profiling/flops_profiler/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/profiling/flops_profiler/README.md -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/profiling/flops_profiler/__init__.py: -------------------------------------------------------------------------------- 1 | from .profiler import * 2 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/activation_checkpointing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/bf16_optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/bf16_optimizer.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/checkpoint_engine/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/comm/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/comm/mpi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/comm/mpi.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/comm/nccl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/comm/nccl.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/compression/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/compression/cupy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/compression/cupy.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/config.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/config_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/config_utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/data_pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/dataloader.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/eigenvalue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/eigenvalue.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/engine.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/fp16/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/fp16/fused_optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/fp16/fused_optimizer.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/fp16/loss_scaler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/fp16/loss_scaler.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/fp16/onebit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/fp16/onebit/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/fp16/onebit/adam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/fp16/onebit/adam.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/fp16/onebit/lamb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/fp16/onebit/lamb.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/fp16/onebit/zoadam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/fp16/onebit/zoadam.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/fp16/unfused_optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/fp16/unfused_optimizer.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/lr_schedules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/lr_schedules.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/pipe/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/pipe/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/pipe/engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/pipe/engine.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/pipe/module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/pipe/module.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/pipe/p2p.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/pipe/p2p.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/pipe/reconfiguration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/pipe/reconfiguration.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/pipe/schedule.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/pipe/schedule.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/pipe/topology.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/pipe/topology.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/progressive_layer_drop.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/progressive_layer_drop.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/quantize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/quantize.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/sparse_tensor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/sparse_tensor.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/state_dict_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/state_dict_factory.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/swap_tensor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/swap_tensor/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/swap_tensor/aio_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/swap_tensor/aio_config.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/swap_tensor/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/swap_tensor/constants.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/swap_tensor/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/swap_tensor/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/weight_quantizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/weight_quantizer.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/zero/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/zero/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/zero/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/zero/config.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/zero/linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/zero/linear.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/zero/offload_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/zero/offload_config.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/zero/parameter_offload.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/zero/parameter_offload.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/zero/stage3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/zero/stage3.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/zero/stage_1_and_2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/zero/stage_1_and_2.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/zero/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/zero/test.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/zero/tiling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/zero/tiling.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/runtime/zero/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/runtime/zero/utils.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/comms_logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/comms_logging.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/debug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/debug.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/exceptions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/exceptions.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/groups.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/groups.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/init_on_device.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/init_on_device.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/logging.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/mixed_precision_linkage.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/mixed_precision_linkage.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/nvtx.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/nvtx.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/tensor_fragment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/tensor_fragment.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/timer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/timer.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/types.py -------------------------------------------------------------------------------- /DeepSpeed/deepspeed/utils/zero_to_fp32.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/deepspeed/utils/zero_to_fp32.py -------------------------------------------------------------------------------- /DeepSpeed/docker/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docker/Dockerfile -------------------------------------------------------------------------------- /DeepSpeed/docker/Dockerfile.rocm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docker/Dockerfile.rocm -------------------------------------------------------------------------------- /DeepSpeed/docs/404.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/404.html -------------------------------------------------------------------------------- /DeepSpeed/docs/CNAME: -------------------------------------------------------------------------------- 1 | www.deepspeed.ai 2 | -------------------------------------------------------------------------------- /DeepSpeed/docs/Gemfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/Gemfile -------------------------------------------------------------------------------- /DeepSpeed/docs/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/README.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_config.yml -------------------------------------------------------------------------------- /DeepSpeed/docs/_data/navigation.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_data/navigation.yml -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/analytics.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/analytics.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/archive-single.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/archive-single.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/author-profile.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/author-profile.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/breadcrumbs.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/breadcrumbs.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/browser-upgrade.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/browser-upgrade.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/category-list.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/category-list.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/comment.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/comment.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/comments.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/comments.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/documents-collection.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/documents-collection.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/feature_row: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/feature_row -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/figure: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/figure -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/footer.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/footer.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/gallery: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/gallery -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/group-by-array: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/group-by-array -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/head.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/head.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/masthead.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/masthead.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/nav_list: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/nav_list -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/page__date.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/page__date.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/page__hero.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/page__hero.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/page__hero_video.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/page__hero_video.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/page__meta.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/page__meta.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/page__taxonomy.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/page__taxonomy.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/paginator.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/paginator.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/post_pagination.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/post_pagination.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/posts-category.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/posts-category.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/posts-tag.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/posts-tag.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/scripts.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/scripts.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/seo.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/seo.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/sidebar.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/sidebar.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/skip-links.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/skip-links.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/social-share.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/social-share.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/tag-list.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/tag-list.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/toc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/toc -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/toc.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/toc.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_includes/video: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_includes/video -------------------------------------------------------------------------------- /DeepSpeed/docs/_layouts/single-full.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_layouts/single-full.html -------------------------------------------------------------------------------- /DeepSpeed/docs/_pages/compression.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_pages/compression.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_pages/config-json.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_pages/config-json.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_pages/inference.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_pages/inference.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_pages/posts-landing.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_pages/posts-landing.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_pages/posts_list_landing.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_pages/posts_list_landing.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_pages/training.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_pages/training.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_pages/tutorials-landing.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_pages/tutorials-landing.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2020-02-13-release.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2020-02-13-release.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2020-02-13-turing-nlg.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2020-02-13-turing-nlg.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2020-03-17-reduce-scatter.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2020-03-17-reduce-scatter.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2020-05-19-bert-record.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2020-05-19-bert-record.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2020-05-19-press-release.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2020-05-19-press-release.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2020-05-19-zero-stage2.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2020-05-19-zero-stage2.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2020-07-24-deepspeed-webinar.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2020-07-24-deepspeed-webinar.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2020-08-07-webinar-on-demand.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2020-08-07-webinar-on-demand.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2020-09-09-ZeRO-Offload.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2020-09-09-ZeRO-Offload.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2020-09-09-onebit-adam-news.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2020-09-09-onebit-adam-news.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2020-09-09-sparse-attention.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2020-09-09-sparse-attention.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2021-03-08-zero3-offload.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2021-03-08-zero3-offload.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2021-05-05-MoQ.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2021-05-05-MoQ.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2021-05-14-inference-release.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2021-05-14-inference-release.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2021-08-18-deepspeed-moe.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2021-08-18-deepspeed-moe.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2021-11-15-autotuning.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2021-11-15-autotuning.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2021-12-09-deepspeed-moe-nlg.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2021-12-09-deepspeed-moe-nlg.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2022-01-19-moe-inference.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2022-01-19-moe-inference.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2022-03-21-amd-support.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2022-03-21-amd-support.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2022-07-26-deepspeed-azure.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2022-07-26-deepspeed-azure.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2022-09-10-zero-inference.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2022-09-10-zero-inference.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_posts/2022-10-11-mii.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_posts/2022-10-11-mii.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_sass/button-group.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_sass/button-group.scss -------------------------------------------------------------------------------- /DeepSpeed/docs/_sass/minimal-mistakes.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_sass/minimal-mistakes.scss -------------------------------------------------------------------------------- /DeepSpeed/docs/_sass/minimal-mistakes/_archive.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_sass/minimal-mistakes/_archive.scss -------------------------------------------------------------------------------- /DeepSpeed/docs/_sass/minimal-mistakes/_navigation.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_sass/minimal-mistakes/_navigation.scss -------------------------------------------------------------------------------- /DeepSpeed/docs/_sass/minimal-mistakes/_page.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_sass/minimal-mistakes/_page.scss -------------------------------------------------------------------------------- /DeepSpeed/docs/_sass/minimal-mistakes/_sidebar.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_sass/minimal-mistakes/_sidebar.scss -------------------------------------------------------------------------------- /DeepSpeed/docs/_sass/minimal-mistakes/_variables.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_sass/minimal-mistakes/_variables.scss -------------------------------------------------------------------------------- /DeepSpeed/docs/_sass/minimal-mistakes/skins/_air.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_sass/minimal-mistakes/skins/_air.scss -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/MoQ-tutorial.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/MoQ-tutorial.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/advanced-install.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/advanced-install.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/autotuning.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/autotuning.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/azure.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/azure.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/bert-finetuning.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/bert-finetuning.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/bert-pretraining.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/bert-pretraining.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/cifar-10.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/cifar-10.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/comms-logging.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/comms-logging.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/curriculum-learning.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/curriculum-learning.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/flops-profiler.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/flops-profiler.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/gan.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/gan.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/getting-started.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/getting-started.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/inference-tutorial.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/inference-tutorial.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/large-models-w-deepspeed.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/large-models-w-deepspeed.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/lrrt.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/lrrt.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/megatron.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/megatron.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/mixture-of-experts-nlg.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/mixture-of-experts-nlg.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/mixture-of-experts.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/mixture-of-experts.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/model-compression.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/model-compression.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/monitor.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/monitor.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/one-cycle.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/one-cycle.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/onebit-adam.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/onebit-adam.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/onebit-lamb.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/onebit-lamb.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/pipeline.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/pipeline.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/pytorch-profiler.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/pytorch-profiler.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/sparse-attention.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/sparse-attention.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/transformer_kernel.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/transformer_kernel.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/zero-offload.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/zero-offload.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/zero-one-adam.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/zero-one-adam.md -------------------------------------------------------------------------------- /DeepSpeed/docs/_tutorials/zero.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/_tutorials/zero.md -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/css/main.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/css/main.scss -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/1.3B-MoE-128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/1.3B-MoE-128.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/175b-trend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/175b-trend.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/1cycle_lr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/1cycle_lr.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/1t-trend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/1t-trend.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/3d-parallelism.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/3d-parallelism.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/3pillars.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/3pillars.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/530b-trend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/530b-trend.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/DeepSpeed-vs-Megatron.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/DeepSpeed-vs-Megatron.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/DeepSpeed_dark.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/DeepSpeed_dark.svg -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/DeepSpeed_light.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/DeepSpeed_light.svg -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/accelerate-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/accelerate-dark.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/accelerate-light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/accelerate-light.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/accelerate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/accelerate.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/adam-convergence.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/adam-convergence.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/bert-ib.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/bert-ib.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/bert-scaling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/bert-scaling.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/bert-tcp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/bert-tcp.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/bingbert-mixedbit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/bingbert-mixedbit.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/convergence-table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/convergence-table.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/deepspeed-speedup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/deepspeed-speedup.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/gpu-numbers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/gpu-numbers.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/hf-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/hf-logo.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/hf-transformers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/hf-transformers.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/inference-latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/inference-latency.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/inference-throughput.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/inference-throughput.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/large-model-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/large-model-graph.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/layernorm_animation.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/layernorm_animation.gif -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/layernorm_deepspeed.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/layernorm_deepspeed.gif -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/layernorm_ds.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/layernorm_ds.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/layernorm_pytorch.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/layernorm_pytorch.gif -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/layernorm_torch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/layernorm_torch.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/lightning-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/lightning-dark.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/lightning-dark.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/lightning-dark.svg -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/lightning-light.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/lightning-light.svg -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/lightning.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/lightning.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/loss_and_lr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/loss_and_lr.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/lr_schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/lr_schedule.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/azure-cost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/azure-cost.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/bert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/bert.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/bloom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/bloom.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/gpt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/gpt.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/hero-transparent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/hero-transparent.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/hero.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/hero.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/mii-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/mii-arch.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/multi-gpu-latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/multi-gpu-latency.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/opt-bloom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/opt-bloom.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/opt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/opt.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/roberta.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/roberta.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/sd-latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/sd-latency.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mii/tput-llms.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mii/tput-llms.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/model_convergence.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/model_convergence.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/moe-lat-tput.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/moe-lat-tput.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/moe-nlg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/moe-nlg.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/mosaicml.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/mosaicml.svg -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/old-vs-new-azure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/old-vs-new-azure.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/onebit-adam-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/onebit-adam-overview.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/onebit-convergence.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/onebit-convergence.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/oom_dp8_1.5B_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/oom_dp8_1.5B_log.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/perf-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/perf-overview.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/pipe-schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/pipe-schedule.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/pp-lowbw-gpt2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/pp-lowbw-gpt2.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/prmoe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/prmoe.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/qkv_fusion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/qkv_fusion.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/quantization-8bit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/quantization-8bit.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/quantization-mixedbit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/quantization-mixedbit.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/sa_backward_pass.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/sa_backward_pass.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/sa_forward_pass.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/sa_forward_pass.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/sa_gpt2_time_result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/sa_gpt2_time_result.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/softmax_animation.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/softmax_animation.gif -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/softmax_deepspeed.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/softmax_deepspeed.gif -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/softmax_ds.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/softmax_ds.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/softmax_pytorch.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/softmax_pytorch.gif -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/softmax_torch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/softmax_torch.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/squad-ib.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/squad-ib.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/squad-scaling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/squad-scaling.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/squad-tcp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/squad-tcp.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/tensorboard_monitor.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/tensorboard_monitor.PNG -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/transformers-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/transformers-dark.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/transformers-light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/transformers-light.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/vmss-setup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/vmss-setup.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/wandb_monitor.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/wandb_monitor.PNG -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/webinar-aug2020.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/webinar-aug2020.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/xtc-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/xtc-1.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/xtc-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/xtc-2.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/xtc-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/xtc-3.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/xtc-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/xtc-4.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/zero-full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/zero-full.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/zero1_dp8_1.5B_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/zero1_dp8_1.5B_log.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/zero1_dp8_1.5B_smi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/zero1_dp8_1.5B_smi.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/zero2_dp32_10B_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/zero2_dp32_10B_log.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/zero2_dp32_10B_smi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/zero2_dp32_10B_smi.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/zero3-offload-1-v100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/zero3-offload-1-v100.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/zero3-offload-16-v100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/zero3-offload-16-v100.png -------------------------------------------------------------------------------- /DeepSpeed/docs/assets/images/zero_inference_models.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/assets/images/zero_inference_models.png -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/Makefile -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/build-api-docs.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/build-api-docs.sh -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/autotuning.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/autotuning.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/conf.py -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/flops-profiler.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/flops-profiler.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/index.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/inference-engine.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/inference-engine.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/inference-init.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/inference-init.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/initialize.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/initialize.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/kernel.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/kernel.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/memory.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/memory.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/moe.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/moe.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/optimizers.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/optimizers.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/pipeline.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/pipeline.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/schedulers.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/schedulers.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/training.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/training.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/code-docs/source/zero3.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/code-docs/source/zero3.rst -------------------------------------------------------------------------------- /DeepSpeed/docs/contributing.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/contributing.md -------------------------------------------------------------------------------- /DeepSpeed/docs/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/docs/index.md -------------------------------------------------------------------------------- /DeepSpeed/examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/examples/README.md -------------------------------------------------------------------------------- /DeepSpeed/install.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/install.sh -------------------------------------------------------------------------------- /DeepSpeed/op_builder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/async_io.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/async_io.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/builder.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/cpu_adagrad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/cpu_adagrad.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/cpu_adam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/cpu_adam.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/fused_adam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/fused_adam.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/fused_lamb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/fused_lamb.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/quantizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/quantizer.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/sparse_attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/sparse_attn.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/stochastic_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/stochastic_transformer.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/transformer.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/transformer_inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/transformer_inference.py -------------------------------------------------------------------------------- /DeepSpeed/op_builder/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/op_builder/utils.py -------------------------------------------------------------------------------- /DeepSpeed/release/bump_patch_version.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/release/bump_patch_version.py -------------------------------------------------------------------------------- /DeepSpeed/release/release.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/release/release.sh -------------------------------------------------------------------------------- /DeepSpeed/requirements/requirements-1bit-mpi.txt: -------------------------------------------------------------------------------- 1 | mpi4py 2 | -------------------------------------------------------------------------------- /DeepSpeed/requirements/requirements-autotuning-ml.txt: -------------------------------------------------------------------------------- 1 | hjson 2 | tabulate 3 | xgboost 4 | -------------------------------------------------------------------------------- /DeepSpeed/requirements/requirements-autotuning.txt: -------------------------------------------------------------------------------- 1 | tabulate 2 | -------------------------------------------------------------------------------- /DeepSpeed/requirements/requirements-dev.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/requirements/requirements-dev.txt -------------------------------------------------------------------------------- /DeepSpeed/requirements/requirements-inf.txt: -------------------------------------------------------------------------------- 1 | lm-eval>=0.2.0 2 | transformers 3 | -------------------------------------------------------------------------------- /DeepSpeed/requirements/requirements-readthedocs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/requirements/requirements-readthedocs.txt -------------------------------------------------------------------------------- /DeepSpeed/requirements/requirements-sd.txt: -------------------------------------------------------------------------------- 1 | diffusers 2 | triton==2.0.0.dev20221005 3 | -------------------------------------------------------------------------------- /DeepSpeed/requirements/requirements-sparse_attn.txt: -------------------------------------------------------------------------------- 1 | triton==1.0.0 2 | -------------------------------------------------------------------------------- /DeepSpeed/requirements/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/requirements/requirements.txt -------------------------------------------------------------------------------- /DeepSpeed/scripts/check-torchdist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/scripts/check-torchdist.py -------------------------------------------------------------------------------- /DeepSpeed/setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/setup.cfg -------------------------------------------------------------------------------- /DeepSpeed/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/setup.py -------------------------------------------------------------------------------- /DeepSpeed/tests/benchmarks/flatten_bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/benchmarks/flatten_bench.py -------------------------------------------------------------------------------- /DeepSpeed/tests/benchmarks/unflatten_bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/benchmarks/unflatten_bench.py -------------------------------------------------------------------------------- /DeepSpeed/tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/conftest.py -------------------------------------------------------------------------------- /DeepSpeed/tests/lightning/test_simple.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/lightning/test_simple.py -------------------------------------------------------------------------------- /DeepSpeed/tests/model/BingBertSquad/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/model/BingBertSquad/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/tests/model/BingBertSquad/run_tests.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/model/BingBertSquad/run_tests.sh -------------------------------------------------------------------------------- /DeepSpeed/tests/model/BingBertSquad/test_e2e_squad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/model/BingBertSquad/test_e2e_squad.py -------------------------------------------------------------------------------- /DeepSpeed/tests/model/Megatron_GPT2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/model/Megatron_GPT2/__init__.py -------------------------------------------------------------------------------- /DeepSpeed/tests/model/Megatron_GPT2/ds_gpt2_test.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/model/Megatron_GPT2/ds_gpt2_test.sh -------------------------------------------------------------------------------- /DeepSpeed/tests/model/Megatron_GPT2/run_func_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/model/Megatron_GPT2/run_func_test.py -------------------------------------------------------------------------------- /DeepSpeed/tests/model/Megatron_GPT2/run_perf_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/model/Megatron_GPT2/run_perf_test.py -------------------------------------------------------------------------------- /DeepSpeed/tests/model/Megatron_GPT2/test_common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/model/Megatron_GPT2/test_common.py -------------------------------------------------------------------------------- /DeepSpeed/tests/model/run_sanity_check.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/model/run_sanity_check.py -------------------------------------------------------------------------------- /DeepSpeed/tests/onebit/test_mpi_backend.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/onebit/test_mpi_backend.py -------------------------------------------------------------------------------- /DeepSpeed/tests/onebit/test_mpi_perf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/onebit/test_mpi_perf.py -------------------------------------------------------------------------------- /DeepSpeed/tests/onebit/test_nccl_backend.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/onebit/test_nccl_backend.py -------------------------------------------------------------------------------- /DeepSpeed/tests/onebit/test_nccl_perf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/onebit/test_nccl_perf.py -------------------------------------------------------------------------------- /DeepSpeed/tests/perf/adam_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/perf/adam_test.py -------------------------------------------------------------------------------- /DeepSpeed/tests/perf/adam_test1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/perf/adam_test1.py -------------------------------------------------------------------------------- /DeepSpeed/tests/pytest.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/pytest.ini -------------------------------------------------------------------------------- /DeepSpeed/tests/small_model_debugging/stage3_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/small_model_debugging/stage3_test.py -------------------------------------------------------------------------------- /DeepSpeed/tests/small_model_debugging/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/small_model_debugging/test.py -------------------------------------------------------------------------------- /DeepSpeed/tests/small_model_debugging/test_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/small_model_debugging/test_model.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/alexnet_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/alexnet_model.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/autotuning/test_autotuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/autotuning/test_autotuning.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/checkpoint/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/checkpoint/common.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/checkpoint/test_lr_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/checkpoint/test_lr_scheduler.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/checkpoint/test_moe_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/checkpoint/test_moe_checkpoint.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/checkpoint/test_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/checkpoint/test_pipeline.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/checkpoint/test_sparse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/checkpoint/test_sparse.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/checkpoint/test_tag_validation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/checkpoint/test_tag_validation.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/checkpoint/test_zero_optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/checkpoint/test_zero_optimizer.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/comm/test_dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/comm/test_dist.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/common.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/compression/test_compression.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/compression/test_compression.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/ds_batch_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/ds_batch_config.json -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/elasticity/test_elastic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/elasticity/test_elastic.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/gpt2-merges.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/gpt2-merges.txt -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/gpt2-vocab.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/gpt2-vocab.json -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/inference/test_inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/inference/test_inference.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/launcher/test_ds_arguments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/launcher/test_ds_arguments.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/launcher/test_multinode_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/launcher/test_multinode_runner.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/launcher/test_run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/launcher/test_run.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/megatron_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/megatron_model.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/modeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/modeling.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/modelingpreln.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/modelingpreln.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/moe/test_moe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/moe/test_moe.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/moe/test_moe_tp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/moe/test_moe_tp.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/monitor/test_monitor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/monitor/test_monitor.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/multi_output_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/multi_output_model.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/ops/adagrad/test_cpu_adagrad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/ops/adagrad/test_cpu_adagrad.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/ops/adam/test_adamw.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/ops/adam/test_adamw.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/ops/adam/test_cpu_adam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/ops/adam/test_cpu_adam.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/ops/aio/test_aio.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/ops/aio/test_aio.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/ops/cuda/test_cuda_backward.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/ops/cuda/test_cuda_backward.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/ops/cuda/test_cuda_forward.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/ops/cuda/test_cuda_forward.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/ops/quantizer/test_quant.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/ops/quantizer/test_quant.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/pipe/test_pipe_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/pipe/test_pipe_module.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/pipe/test_pipe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/pipe/test_pipe.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/pipe/test_topology.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/pipe/test_topology.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/sparse_tensor/test_csr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/sparse_tensor/test_csr.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/test_autocast.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/test_autocast.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/test_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/test_data.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/test_ds_config_dict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/test_ds_config_dict.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/test_ds_config_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/test_ds_config_model.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/test_ds_initialize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/test_ds_initialize.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/test_lr_schedulers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/test_lr_schedulers.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/test_pld.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/test_pld.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/test_runtime_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/test_runtime_utils.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/utils/test_partition.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/utils/test_partition.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/zero/test_zero.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/zero/test_zero.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/zero/test_zero_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/zero/test_zero_config.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/zero/test_zero_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/zero/test_zero_context.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/runtime/zero/test_zero_tiled.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/runtime/zero/test_zero_tiled.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/simple_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/simple_model.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/util.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/utils/test_get_optim_files.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/utils/test_get_optim_files.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/utils/test_groups.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/utils/test_groups.py -------------------------------------------------------------------------------- /DeepSpeed/tests/unit/utils/test_init_on_device.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/DeepSpeed/tests/unit/utils/test_init_on_device.py -------------------------------------------------------------------------------- /DeepSpeed/version.txt: -------------------------------------------------------------------------------- 1 | 0.7.4 2 | -------------------------------------------------------------------------------- /Megatron-DeepSpeed/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/.gitignore -------------------------------------------------------------------------------- /Megatron-DeepSpeed/CODEOWNERS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/CODEOWNERS -------------------------------------------------------------------------------- /Megatron-DeepSpeed/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/LICENSE -------------------------------------------------------------------------------- /Megatron-DeepSpeed/MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/MANIFEST.in -------------------------------------------------------------------------------- /Megatron-DeepSpeed/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/README.md -------------------------------------------------------------------------------- /Megatron-DeepSpeed/SECURITY.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/SECURITY.md -------------------------------------------------------------------------------- /Megatron-DeepSpeed/dataset/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/dataset/README.md -------------------------------------------------------------------------------- /Megatron-DeepSpeed/dataset/download_books.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/dataset/download_books.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/dataset/download_ckpt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/dataset/download_ckpt.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/dataset/download_vocab.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/dataset/download_vocab.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/MoE/ds_evalharness.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/MoE/ds_evalharness.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/MoE/profile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/MoE/profile.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/MoE/readme_evalharness.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/MoE/readme_evalharness.md -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/README.md -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/azure/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/azure/README.md -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/azure/run-175b.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/azure/run-175b.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/azure/run-1t.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/azure/run-1t.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/azureml/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/azureml/README.md -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/azureml/aml_submit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/azureml/aml_submit.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/azureml/prepare_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/azureml/prepare_dataset.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/create_embeddings.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/create_embeddings.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/generate_text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/generate_text.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/merge_mp_bert.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/merge_mp_bert.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/pretrain_bert.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/pretrain_bert.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/pretrain_gpt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/pretrain_gpt.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/pretrain_gpt3_175B.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/pretrain_gpt3_175B.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/pretrain_ict.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/pretrain_ict.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/examples/pretrain_t5.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/examples/pretrain_t5.sh -------------------------------------------------------------------------------- /Megatron-DeepSpeed/images/cases_april2021.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/images/cases_april2021.png -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/__init__.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/arguments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/arguments.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/checkpointing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/checkpointing.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/Makefile -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/__init__.py: -------------------------------------------------------------------------------- 1 | from . import indexed_dataset 2 | -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/autoaugment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/autoaugment.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/bert_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/bert_dataset.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/data_samplers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/data_samplers.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/dataset_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/dataset_utils.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/gpt_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/gpt_dataset.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/helpers.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/helpers.cpp -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/ict_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/ict_dataset.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/indexed_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/indexed_dataset.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/realm_index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/realm_index.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/t5_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/t5_dataset.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/data/vit_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/data/vit_dataset.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/enums.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/enums.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/fused_kernels/compat.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/global_vars.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/global_vars.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/indexer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/indexer.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/initialize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/initialize.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/learning_rates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/learning_rates.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/memory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/memory.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/microbatches.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/microbatches.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/__init__.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/bert_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/bert_model.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/classification.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/distributed.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/enums.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/enums.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/fused_softmax.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/fused_softmax.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/gpt_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/gpt_model.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/language_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/language_model.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/module.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/realm_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/realm_model.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/t5_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/t5_model.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/transformer.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/utils.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/model/vit_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/model/vit_model.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/mpu/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/mpu/__init__.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/mpu/cross_entropy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/mpu/cross_entropy.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/mpu/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/mpu/data.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/mpu/initialize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/mpu/initialize.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/mpu/layers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/mpu/layers.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/mpu/mappings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/mpu/mappings.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/mpu/random.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/mpu/random.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/mpu/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/mpu/tests/commons.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/mpu/tests/commons.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/mpu/tests/test_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/mpu/tests/test_data.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/mpu/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/mpu/utils.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/optimizer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/optimizer/__init__.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/optimizer/clip_grads.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/optimizer/clip_grads.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/optimizer/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/optimizer/optimizer.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/p2p_communication.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/p2p_communication.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/package_info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/package_info.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/schedules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/schedules.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/tokenizer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/tokenizer/__init__.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/tokenizer/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/tokenizer/tokenizer.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/training.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/megatron/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/megatron/utils.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/pretrain_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/pretrain_bert.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/pretrain_gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/pretrain_gpt.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/pretrain_ict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/pretrain_ict.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/pretrain_t5.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/pretrain_t5.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/pretrain_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/pretrain_vit.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/requirements.txt: -------------------------------------------------------------------------------- 1 | pybind11 2 | torch 3 | six 4 | regex 5 | numpy==1.20.0 6 | -------------------------------------------------------------------------------- /Megatron-DeepSpeed/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/setup.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/data_utils.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/ensemble_classifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/ensemble_classifier.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/eval_harness/download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/eval_harness/download.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/eval_harness/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/eval_harness/evaluate.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/eval_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/eval_utils.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/finetune_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/finetune_utils.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/glue/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/glue/data.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/glue/finetune.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/glue/finetune.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/glue/mnli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/glue/mnli.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/glue/qqp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/glue/qqp.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/main.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/orqa/evaluate_orqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/orqa/evaluate_orqa.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/orqa/evaluate_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/orqa/evaluate_utils.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/race/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/race/data.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/race/finetune.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/race/finetune.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/vision/classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/vision/classification.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/vision/eval_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/vision/eval_utils.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/vision/finetune_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/vision/finetune_utils.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/vision/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/vision/main.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/zeroshot_gpt/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/zeroshot_gpt/datasets.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tasks/zeroshot_gpt/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tasks/zeroshot_gpt/evaluate.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tests/test_basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tests/test_basic.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tools/create_doc_index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tools/create_doc_index.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tools/generate_samples_gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tools/generate_samples_gpt.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tools/linter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tools/linter.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tools/merge_mp_partitions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tools/merge_mp_partitions.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tools/openwebtext/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tools/openwebtext/README.md -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tools/openwebtext/add_id.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tools/openwebtext/add_id.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tools/openwebtext/merge_jsons.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tools/openwebtext/merge_jsons.py -------------------------------------------------------------------------------- /Megatron-DeepSpeed/tools/preprocess_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/Megatron-DeepSpeed/tools/preprocess_data.py -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/README.md -------------------------------------------------------------------------------- /benchmark/multi_node/model/gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/multi_node/model/gpt.py -------------------------------------------------------------------------------- /benchmark/multi_node/script/config/TEMPLATE.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/multi_node/script/config/TEMPLATE.json -------------------------------------------------------------------------------- /benchmark/multi_node/script/fig12.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/multi_node/script/fig12.sh -------------------------------------------------------------------------------- /benchmark/multi_node/script/run_gpt_350M.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/multi_node/script/run_gpt_350M.sh -------------------------------------------------------------------------------- /benchmark/multi_node/script/run_megatron_1.3B.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/multi_node/script/run_megatron_1.3B.sh -------------------------------------------------------------------------------- /benchmark/multi_node/script/run_megatron_760M.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/multi_node/script/run_megatron_760M.sh -------------------------------------------------------------------------------- /benchmark/single_node/model/bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/model/bert.py -------------------------------------------------------------------------------- /benchmark/single_node/model/gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/model/gpt.py -------------------------------------------------------------------------------- /benchmark/single_node/model/resnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/model/resnet.py -------------------------------------------------------------------------------- /benchmark/single_node/script/config/TEMPLATE.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/config/TEMPLATE.json -------------------------------------------------------------------------------- /benchmark/single_node/script/fig10.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/fig10.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/fig11.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/fig11.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/fig13.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/fig13.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/fig9_a.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/fig9_a.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/fig9_b.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/fig9_b.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/run_bert_1.3B.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/run_bert_1.3B.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/run_bert_3.9B.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/run_bert_3.9B.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/run_bert_336M.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/run_bert_336M.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/run_gpt_125M.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/run_gpt_125M.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/run_gpt_350M.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/run_gpt_350M.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/run_megatron_125M.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/run_megatron_125M.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/run_megatron_350M.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/run_megatron_350M.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/run_megatron_760M.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/run_megatron_760M.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/run_resnet_152.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/run_resnet_152.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/single_3090.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/single_3090.sh -------------------------------------------------------------------------------- /benchmark/single_node/script/single_v100.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/casys-kaist/EnvPipe/HEAD/benchmark/single_node/script/single_v100.sh --------------------------------------------------------------------------------