├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.yml │ ├── build_err.yml │ └── features_request.yml ├── pull_request_template.md └── workflows │ ├── build.yml │ ├── build_whl.yml │ ├── publish.yaml │ └── release.yml ├── .gitignore ├── CMakeLists.txt ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README-ZH.md ├── README.md ├── Release.txt ├── bmtrain ├── __init__.py ├── benchmark │ ├── __init__.py │ ├── all_gather.py │ ├── reduce_scatter.py │ ├── send_recv.py │ ├── shape.py │ └── utils.py ├── block_layer.py ├── debug.py ├── distributed │ ├── __init__.py │ └── ops.py ├── global_var.py ├── hook_func.py ├── init.py ├── inspect │ ├── __init__.py │ ├── format.py │ ├── model.py │ └── tensor.py ├── layer.py ├── loss │ ├── __init__.py │ ├── _function.py │ └── cross_entropy.py ├── lr_scheduler │ ├── __init__.py │ ├── cosine.py │ ├── exponential.py │ ├── linear.py │ ├── no_decay.py │ ├── noam.py │ └── warmup.py ├── nccl │ ├── __init__.py │ └── enums.py ├── nn │ ├── __init__.py │ ├── column_parallel_linear.py │ ├── linear.py │ ├── parallel_embedding.py │ ├── parallel_linear_func.py │ └── row_parallel_linear.py ├── optim │ ├── __init__.py │ ├── _distributed.py │ ├── _function.py │ ├── adam.py │ ├── adam_offload.py │ └── optim_manager.py ├── param_init.py ├── parameter.py ├── pipe_layer.py ├── store.py ├── synchronize.py ├── utils.py ├── wrapper.py └── zero_context.py ├── cmake └── FindNCCL.cmake ├── csrc ├── bind.cpp ├── cuda │ ├── adam_cuda.cu │ ├── bfloat16.cuh │ ├── cross_entropy.cu │ ├── has_inf_nan.cu │ └── reduce.cuh └── include │ ├── adam_cpu.hpp │ ├── bind.hpp │ ├── cpu_info.h │ └── nccl.hpp ├── doc_requirements.txt ├── docs ├── Makefile ├── UPDATE_0.2.0.md ├── UPDATE_0.2.3.md ├── UPDATE_1.0.0.md ├── logo.png ├── make.bat ├── source-en │ ├── _static │ │ ├── css │ │ │ └── custom.css │ │ └── js │ │ │ └── custom.js │ ├── api │ │ ├── bmtrain.benchmark.rst_bk │ │ ├── bmtrain.distributed.rst_bk │ │ ├── bmtrain.inspect.rst │ │ ├── bmtrain.loss.rst │ │ ├── bmtrain.lr_scheduler.rst │ │ ├── bmtrain.nccl.rst_bk │ │ ├── bmtrain.nn.rst │ │ ├── bmtrain.optim.rst │ │ ├── bmtrain.rst │ │ └── modules.rst │ ├── conf.py │ ├── index.rst │ └── notes │ │ ├── image │ │ ├── ZeRO3.png │ │ ├── communication_example.png │ │ ├── communication_fig.png │ │ ├── cpu.png │ │ └── zero3_example.png │ │ ├── installation.md │ │ ├── quickstart.md │ │ └── tech.md └── source │ ├── _static │ ├── css │ │ └── custom.css │ └── js │ │ └── custom.js │ ├── api │ ├── bmtrain.benchmark.rst_bk │ ├── bmtrain.distributed.rst_bk │ ├── bmtrain.inspect.rst │ ├── bmtrain.loss.rst │ ├── bmtrain.lr_scheduler.rst │ ├── bmtrain.nccl.rst_bk │ ├── bmtrain.nn.rst │ ├── bmtrain.optim.rst │ ├── bmtrain.rst │ └── modules.rst │ ├── conf.py │ ├── index.rst │ └── notes │ ├── image │ ├── ZeRO3.png │ ├── communication_example.png │ ├── communication_fig.png │ ├── cpu.png │ └── zero3_example.png │ ├── installation.md │ ├── quickstart.md │ └── tech.md ├── example ├── README.md ├── benchmark.py ├── layers │ ├── __init__.py │ ├── attention.py │ ├── embedding.py │ ├── feedforward.py │ ├── layernorm.py │ └── transformer.py ├── models │ ├── __init__.py │ └── gpt.py ├── run.sh ├── sbatch.sh └── train.py ├── other_requirements.txt ├── pyproject.toml ├── setup.py └── tests ├── test_all.py ├── test_column_parallel_linear.py ├── test_different_output_shape.py ├── test_dropout.py ├── test_grad_accu.py ├── test_has_inf_nan.py ├── test_init_parameters.py ├── test_init_parameters_multi_gpu.py ├── test_inspector_hidden.py ├── test_load_ckpt.py ├── test_loss_func.py ├── test_middle_hidden.py ├── test_model_wrapper.py ├── test_multi_return.py ├── test_nccl_backward.py ├── test_no_grad.py ├── test_optim.py ├── test_optim_state.py ├── test_other_hidden.py ├── test_parallel_projection.py ├── test_requires_grad.py ├── test_requires_grad_multi_gpu.py ├── test_row_parallel_linear.py ├── test_send_recv.py ├── test_store.py ├── test_synchronize.py ├── test_training.py └── utils.py /.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/.dockerignore -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/.github/ISSUE_TEMPLATE/bug_report.yml -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/build_err.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/.github/ISSUE_TEMPLATE/build_err.yml -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/features_request.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/.github/ISSUE_TEMPLATE/features_request.yml -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/.github/pull_request_template.md -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/.github/workflows/build.yml -------------------------------------------------------------------------------- /.github/workflows/build_whl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/.github/workflows/build_whl.yml -------------------------------------------------------------------------------- /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/.github/workflows/publish.yaml -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/.github/workflows/release.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/.gitignore -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/CMakeLists.txt -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README-ZH.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/README-ZH.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/README.md -------------------------------------------------------------------------------- /Release.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/Release.txt -------------------------------------------------------------------------------- /bmtrain/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/__init__.py -------------------------------------------------------------------------------- /bmtrain/benchmark/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/benchmark/__init__.py -------------------------------------------------------------------------------- /bmtrain/benchmark/all_gather.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/benchmark/all_gather.py -------------------------------------------------------------------------------- /bmtrain/benchmark/reduce_scatter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/benchmark/reduce_scatter.py -------------------------------------------------------------------------------- /bmtrain/benchmark/send_recv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/benchmark/send_recv.py -------------------------------------------------------------------------------- /bmtrain/benchmark/shape.py: -------------------------------------------------------------------------------- 1 | SHAPES = [ 2 | (2**i) for i in range(10, 33) 3 | ] -------------------------------------------------------------------------------- /bmtrain/benchmark/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/benchmark/utils.py -------------------------------------------------------------------------------- /bmtrain/block_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/block_layer.py -------------------------------------------------------------------------------- /bmtrain/debug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/debug.py -------------------------------------------------------------------------------- /bmtrain/distributed/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/distributed/__init__.py -------------------------------------------------------------------------------- /bmtrain/distributed/ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/distributed/ops.py -------------------------------------------------------------------------------- /bmtrain/global_var.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/global_var.py -------------------------------------------------------------------------------- /bmtrain/hook_func.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/hook_func.py -------------------------------------------------------------------------------- /bmtrain/init.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/init.py -------------------------------------------------------------------------------- /bmtrain/inspect/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/inspect/__init__.py -------------------------------------------------------------------------------- /bmtrain/inspect/format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/inspect/format.py -------------------------------------------------------------------------------- /bmtrain/inspect/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/inspect/model.py -------------------------------------------------------------------------------- /bmtrain/inspect/tensor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/inspect/tensor.py -------------------------------------------------------------------------------- /bmtrain/layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/layer.py -------------------------------------------------------------------------------- /bmtrain/loss/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/loss/__init__.py -------------------------------------------------------------------------------- /bmtrain/loss/_function.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/loss/_function.py -------------------------------------------------------------------------------- /bmtrain/loss/cross_entropy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/loss/cross_entropy.py -------------------------------------------------------------------------------- /bmtrain/lr_scheduler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/lr_scheduler/__init__.py -------------------------------------------------------------------------------- /bmtrain/lr_scheduler/cosine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/lr_scheduler/cosine.py -------------------------------------------------------------------------------- /bmtrain/lr_scheduler/exponential.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/lr_scheduler/exponential.py -------------------------------------------------------------------------------- /bmtrain/lr_scheduler/linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/lr_scheduler/linear.py -------------------------------------------------------------------------------- /bmtrain/lr_scheduler/no_decay.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/lr_scheduler/no_decay.py -------------------------------------------------------------------------------- /bmtrain/lr_scheduler/noam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/lr_scheduler/noam.py -------------------------------------------------------------------------------- /bmtrain/lr_scheduler/warmup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/lr_scheduler/warmup.py -------------------------------------------------------------------------------- /bmtrain/nccl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/nccl/__init__.py -------------------------------------------------------------------------------- /bmtrain/nccl/enums.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/nccl/enums.py -------------------------------------------------------------------------------- /bmtrain/nn/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/nn/__init__.py -------------------------------------------------------------------------------- /bmtrain/nn/column_parallel_linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/nn/column_parallel_linear.py -------------------------------------------------------------------------------- /bmtrain/nn/linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/nn/linear.py -------------------------------------------------------------------------------- /bmtrain/nn/parallel_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/nn/parallel_embedding.py -------------------------------------------------------------------------------- /bmtrain/nn/parallel_linear_func.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/nn/parallel_linear_func.py -------------------------------------------------------------------------------- /bmtrain/nn/row_parallel_linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/nn/row_parallel_linear.py -------------------------------------------------------------------------------- /bmtrain/optim/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/optim/__init__.py -------------------------------------------------------------------------------- /bmtrain/optim/_distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/optim/_distributed.py -------------------------------------------------------------------------------- /bmtrain/optim/_function.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/optim/_function.py -------------------------------------------------------------------------------- /bmtrain/optim/adam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/optim/adam.py -------------------------------------------------------------------------------- /bmtrain/optim/adam_offload.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/optim/adam_offload.py -------------------------------------------------------------------------------- /bmtrain/optim/optim_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/optim/optim_manager.py -------------------------------------------------------------------------------- /bmtrain/param_init.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/param_init.py -------------------------------------------------------------------------------- /bmtrain/parameter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/parameter.py -------------------------------------------------------------------------------- /bmtrain/pipe_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/pipe_layer.py -------------------------------------------------------------------------------- /bmtrain/store.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/store.py -------------------------------------------------------------------------------- /bmtrain/synchronize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/synchronize.py -------------------------------------------------------------------------------- /bmtrain/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/utils.py -------------------------------------------------------------------------------- /bmtrain/wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/wrapper.py -------------------------------------------------------------------------------- /bmtrain/zero_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/bmtrain/zero_context.py -------------------------------------------------------------------------------- /cmake/FindNCCL.cmake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/cmake/FindNCCL.cmake -------------------------------------------------------------------------------- /csrc/bind.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/csrc/bind.cpp -------------------------------------------------------------------------------- /csrc/cuda/adam_cuda.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/csrc/cuda/adam_cuda.cu -------------------------------------------------------------------------------- /csrc/cuda/bfloat16.cuh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/csrc/cuda/bfloat16.cuh -------------------------------------------------------------------------------- /csrc/cuda/cross_entropy.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/csrc/cuda/cross_entropy.cu -------------------------------------------------------------------------------- /csrc/cuda/has_inf_nan.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/csrc/cuda/has_inf_nan.cu -------------------------------------------------------------------------------- /csrc/cuda/reduce.cuh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/csrc/cuda/reduce.cuh -------------------------------------------------------------------------------- /csrc/include/adam_cpu.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/csrc/include/adam_cpu.hpp -------------------------------------------------------------------------------- /csrc/include/bind.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/csrc/include/bind.hpp -------------------------------------------------------------------------------- /csrc/include/cpu_info.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/csrc/include/cpu_info.h -------------------------------------------------------------------------------- /csrc/include/nccl.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/csrc/include/nccl.hpp -------------------------------------------------------------------------------- /doc_requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/doc_requirements.txt -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/UPDATE_0.2.0.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/UPDATE_0.2.0.md -------------------------------------------------------------------------------- /docs/UPDATE_0.2.3.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/UPDATE_0.2.3.md -------------------------------------------------------------------------------- /docs/UPDATE_1.0.0.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/UPDATE_1.0.0.md -------------------------------------------------------------------------------- /docs/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/logo.png -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/make.bat -------------------------------------------------------------------------------- /docs/source-en/_static/css/custom.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/_static/css/custom.css -------------------------------------------------------------------------------- /docs/source-en/_static/js/custom.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/_static/js/custom.js -------------------------------------------------------------------------------- /docs/source-en/api/bmtrain.benchmark.rst_bk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/api/bmtrain.benchmark.rst_bk -------------------------------------------------------------------------------- /docs/source-en/api/bmtrain.distributed.rst_bk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/api/bmtrain.distributed.rst_bk -------------------------------------------------------------------------------- /docs/source-en/api/bmtrain.inspect.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/api/bmtrain.inspect.rst -------------------------------------------------------------------------------- /docs/source-en/api/bmtrain.loss.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/api/bmtrain.loss.rst -------------------------------------------------------------------------------- /docs/source-en/api/bmtrain.lr_scheduler.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/api/bmtrain.lr_scheduler.rst -------------------------------------------------------------------------------- /docs/source-en/api/bmtrain.nccl.rst_bk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/api/bmtrain.nccl.rst_bk -------------------------------------------------------------------------------- /docs/source-en/api/bmtrain.nn.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/api/bmtrain.nn.rst -------------------------------------------------------------------------------- /docs/source-en/api/bmtrain.optim.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/api/bmtrain.optim.rst -------------------------------------------------------------------------------- /docs/source-en/api/bmtrain.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/api/bmtrain.rst -------------------------------------------------------------------------------- /docs/source-en/api/modules.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/api/modules.rst -------------------------------------------------------------------------------- /docs/source-en/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/conf.py -------------------------------------------------------------------------------- /docs/source-en/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/index.rst -------------------------------------------------------------------------------- /docs/source-en/notes/image/ZeRO3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/notes/image/ZeRO3.png -------------------------------------------------------------------------------- /docs/source-en/notes/image/communication_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/notes/image/communication_example.png -------------------------------------------------------------------------------- /docs/source-en/notes/image/communication_fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/notes/image/communication_fig.png -------------------------------------------------------------------------------- /docs/source-en/notes/image/cpu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/notes/image/cpu.png -------------------------------------------------------------------------------- /docs/source-en/notes/image/zero3_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/notes/image/zero3_example.png -------------------------------------------------------------------------------- /docs/source-en/notes/installation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/notes/installation.md -------------------------------------------------------------------------------- /docs/source-en/notes/quickstart.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/notes/quickstart.md -------------------------------------------------------------------------------- /docs/source-en/notes/tech.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source-en/notes/tech.md -------------------------------------------------------------------------------- /docs/source/_static/css/custom.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/_static/css/custom.css -------------------------------------------------------------------------------- /docs/source/_static/js/custom.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/_static/js/custom.js -------------------------------------------------------------------------------- /docs/source/api/bmtrain.benchmark.rst_bk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/api/bmtrain.benchmark.rst_bk -------------------------------------------------------------------------------- /docs/source/api/bmtrain.distributed.rst_bk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/api/bmtrain.distributed.rst_bk -------------------------------------------------------------------------------- /docs/source/api/bmtrain.inspect.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/api/bmtrain.inspect.rst -------------------------------------------------------------------------------- /docs/source/api/bmtrain.loss.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/api/bmtrain.loss.rst -------------------------------------------------------------------------------- /docs/source/api/bmtrain.lr_scheduler.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/api/bmtrain.lr_scheduler.rst -------------------------------------------------------------------------------- /docs/source/api/bmtrain.nccl.rst_bk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/api/bmtrain.nccl.rst_bk -------------------------------------------------------------------------------- /docs/source/api/bmtrain.nn.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/api/bmtrain.nn.rst -------------------------------------------------------------------------------- /docs/source/api/bmtrain.optim.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/api/bmtrain.optim.rst -------------------------------------------------------------------------------- /docs/source/api/bmtrain.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/api/bmtrain.rst -------------------------------------------------------------------------------- /docs/source/api/modules.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/api/modules.rst -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/conf.py -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/index.rst -------------------------------------------------------------------------------- /docs/source/notes/image/ZeRO3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/notes/image/ZeRO3.png -------------------------------------------------------------------------------- /docs/source/notes/image/communication_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/notes/image/communication_example.png -------------------------------------------------------------------------------- /docs/source/notes/image/communication_fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/notes/image/communication_fig.png -------------------------------------------------------------------------------- /docs/source/notes/image/cpu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/notes/image/cpu.png -------------------------------------------------------------------------------- /docs/source/notes/image/zero3_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/notes/image/zero3_example.png -------------------------------------------------------------------------------- /docs/source/notes/installation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/notes/installation.md -------------------------------------------------------------------------------- /docs/source/notes/quickstart.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/notes/quickstart.md -------------------------------------------------------------------------------- /docs/source/notes/tech.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/docs/source/notes/tech.md -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/README.md -------------------------------------------------------------------------------- /example/benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/benchmark.py -------------------------------------------------------------------------------- /example/layers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/layers/__init__.py -------------------------------------------------------------------------------- /example/layers/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/layers/attention.py -------------------------------------------------------------------------------- /example/layers/embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/layers/embedding.py -------------------------------------------------------------------------------- /example/layers/feedforward.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/layers/feedforward.py -------------------------------------------------------------------------------- /example/layers/layernorm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/layers/layernorm.py -------------------------------------------------------------------------------- /example/layers/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/layers/transformer.py -------------------------------------------------------------------------------- /example/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .gpt import GPT -------------------------------------------------------------------------------- /example/models/gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/models/gpt.py -------------------------------------------------------------------------------- /example/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/run.sh -------------------------------------------------------------------------------- /example/sbatch.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/sbatch.sh -------------------------------------------------------------------------------- /example/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/example/train.py -------------------------------------------------------------------------------- /other_requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/other_requirements.txt -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/pyproject.toml -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/setup.py -------------------------------------------------------------------------------- /tests/test_all.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_all.py -------------------------------------------------------------------------------- /tests/test_column_parallel_linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_column_parallel_linear.py -------------------------------------------------------------------------------- /tests/test_different_output_shape.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_different_output_shape.py -------------------------------------------------------------------------------- /tests/test_dropout.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_dropout.py -------------------------------------------------------------------------------- /tests/test_grad_accu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_grad_accu.py -------------------------------------------------------------------------------- /tests/test_has_inf_nan.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_has_inf_nan.py -------------------------------------------------------------------------------- /tests/test_init_parameters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_init_parameters.py -------------------------------------------------------------------------------- /tests/test_init_parameters_multi_gpu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_init_parameters_multi_gpu.py -------------------------------------------------------------------------------- /tests/test_inspector_hidden.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_inspector_hidden.py -------------------------------------------------------------------------------- /tests/test_load_ckpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_load_ckpt.py -------------------------------------------------------------------------------- /tests/test_loss_func.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_loss_func.py -------------------------------------------------------------------------------- /tests/test_middle_hidden.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_middle_hidden.py -------------------------------------------------------------------------------- /tests/test_model_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_model_wrapper.py -------------------------------------------------------------------------------- /tests/test_multi_return.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_multi_return.py -------------------------------------------------------------------------------- /tests/test_nccl_backward.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_nccl_backward.py -------------------------------------------------------------------------------- /tests/test_no_grad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_no_grad.py -------------------------------------------------------------------------------- /tests/test_optim.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_optim.py -------------------------------------------------------------------------------- /tests/test_optim_state.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_optim_state.py -------------------------------------------------------------------------------- /tests/test_other_hidden.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_other_hidden.py -------------------------------------------------------------------------------- /tests/test_parallel_projection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_parallel_projection.py -------------------------------------------------------------------------------- /tests/test_requires_grad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_requires_grad.py -------------------------------------------------------------------------------- /tests/test_requires_grad_multi_gpu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_requires_grad_multi_gpu.py -------------------------------------------------------------------------------- /tests/test_row_parallel_linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_row_parallel_linear.py -------------------------------------------------------------------------------- /tests/test_send_recv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_send_recv.py -------------------------------------------------------------------------------- /tests/test_store.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_store.py -------------------------------------------------------------------------------- /tests/test_synchronize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_synchronize.py -------------------------------------------------------------------------------- /tests/test_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/test_training.py -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTrain/HEAD/tests/utils.py --------------------------------------------------------------------------------