├── .gitignore ├── LICENSE ├── README.md ├── comm ├── __init__.py ├── comm_utils.py └── nccl_backend.py ├── compress ├── __init__.py ├── delta_modules.py ├── dummy_modules.py ├── fixpoint.py ├── fixpoint_modules.py ├── flag.py └── utils.py ├── convert_deberta_checkpoint.py ├── convert_gpt2_checkpoint.py ├── data_parallel ├── __init__.py ├── dist_dp_allreduce.py ├── dist_dp_central_ps.py ├── dist_dp_sharded_ps.py ├── dist_dp_sharded_ps_compressed.py ├── dist_dp_utils.py └── flatten_utils.py ├── dist_deberta_runner.py ├── dist_lm_runner.py ├── metrics ├── README.md ├── accuracy │ └── accuracy.py ├── f1 │ └── f1.py ├── matthews_correlation │ └── matthews_correlation.py ├── pearsonr │ └── pearsonr.py ├── perplexity │ ├── README.md │ └── perplexity.py ├── perplexity_custom │ └── perplexity_custom.py ├── precision │ └── precision.py └── recall │ └── recall.py ├── modules ├── __init__.py ├── deberta_modules.py ├── dist_deberta_pp_module.py ├── dist_gpt_pp_module.py ├── gpt_modules.py └── tokenizer.py ├── optimizer ├── __init__.py ├── grad_scalar.py └── optimizer.py ├── pipeline_parallel ├── __init__.py ├── dist_gpipe_pipeline_async.py └── dist_pp_utils.py ├── run_deberta.sh ├── run_lm.sh ├── tasks ├── __init__.py └── data_loaders │ ├── __init__.py │ ├── arxiv21.py │ ├── cola.py │ ├── qnli.py │ └── wikitext.py ├── trace_json └── __dummy_file__ └── utils ├── __init__.py ├── dist_args_utils.py ├── dist_debug_utils.py ├── dist_test_utils.py └── dist_train_utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/README.md -------------------------------------------------------------------------------- /comm/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /comm/comm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/comm/comm_utils.py -------------------------------------------------------------------------------- /comm/nccl_backend.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/comm/nccl_backend.py -------------------------------------------------------------------------------- /compress/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/compress/__init__.py -------------------------------------------------------------------------------- /compress/delta_modules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/compress/delta_modules.py -------------------------------------------------------------------------------- /compress/dummy_modules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/compress/dummy_modules.py -------------------------------------------------------------------------------- /compress/fixpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/compress/fixpoint.py -------------------------------------------------------------------------------- /compress/fixpoint_modules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/compress/fixpoint_modules.py -------------------------------------------------------------------------------- /compress/flag.py: -------------------------------------------------------------------------------- 1 | FLAG_DISABLE_COMPRESSION = False -------------------------------------------------------------------------------- /compress/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/compress/utils.py -------------------------------------------------------------------------------- /convert_deberta_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/convert_deberta_checkpoint.py -------------------------------------------------------------------------------- /convert_gpt2_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/convert_gpt2_checkpoint.py -------------------------------------------------------------------------------- /data_parallel/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data_parallel/dist_dp_allreduce.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/data_parallel/dist_dp_allreduce.py -------------------------------------------------------------------------------- /data_parallel/dist_dp_central_ps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/data_parallel/dist_dp_central_ps.py -------------------------------------------------------------------------------- /data_parallel/dist_dp_sharded_ps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/data_parallel/dist_dp_sharded_ps.py -------------------------------------------------------------------------------- /data_parallel/dist_dp_sharded_ps_compressed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/data_parallel/dist_dp_sharded_ps_compressed.py -------------------------------------------------------------------------------- /data_parallel/dist_dp_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/data_parallel/dist_dp_utils.py -------------------------------------------------------------------------------- /data_parallel/flatten_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/data_parallel/flatten_utils.py -------------------------------------------------------------------------------- /dist_deberta_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/dist_deberta_runner.py -------------------------------------------------------------------------------- /dist_lm_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/dist_lm_runner.py -------------------------------------------------------------------------------- /metrics/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/metrics/README.md -------------------------------------------------------------------------------- /metrics/accuracy/accuracy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/metrics/accuracy/accuracy.py -------------------------------------------------------------------------------- /metrics/f1/f1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/metrics/f1/f1.py -------------------------------------------------------------------------------- /metrics/matthews_correlation/matthews_correlation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/metrics/matthews_correlation/matthews_correlation.py -------------------------------------------------------------------------------- /metrics/pearsonr/pearsonr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/metrics/pearsonr/pearsonr.py -------------------------------------------------------------------------------- /metrics/perplexity/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/metrics/perplexity/README.md -------------------------------------------------------------------------------- /metrics/perplexity/perplexity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/metrics/perplexity/perplexity.py -------------------------------------------------------------------------------- /metrics/perplexity_custom/perplexity_custom.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/metrics/perplexity_custom/perplexity_custom.py -------------------------------------------------------------------------------- /metrics/precision/precision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/metrics/precision/precision.py -------------------------------------------------------------------------------- /metrics/recall/recall.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/metrics/recall/recall.py -------------------------------------------------------------------------------- /modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /modules/deberta_modules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/modules/deberta_modules.py -------------------------------------------------------------------------------- /modules/dist_deberta_pp_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/modules/dist_deberta_pp_module.py -------------------------------------------------------------------------------- /modules/dist_gpt_pp_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/modules/dist_gpt_pp_module.py -------------------------------------------------------------------------------- /modules/gpt_modules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/modules/gpt_modules.py -------------------------------------------------------------------------------- /modules/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/modules/tokenizer.py -------------------------------------------------------------------------------- /optimizer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /optimizer/grad_scalar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/optimizer/grad_scalar.py -------------------------------------------------------------------------------- /optimizer/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/optimizer/optimizer.py -------------------------------------------------------------------------------- /pipeline_parallel/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipeline_parallel/dist_gpipe_pipeline_async.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/pipeline_parallel/dist_gpipe_pipeline_async.py -------------------------------------------------------------------------------- /pipeline_parallel/dist_pp_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/pipeline_parallel/dist_pp_utils.py -------------------------------------------------------------------------------- /run_deberta.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/run_deberta.sh -------------------------------------------------------------------------------- /run_lm.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/run_lm.sh -------------------------------------------------------------------------------- /tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tasks/data_loaders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tasks/data_loaders/arxiv21.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/tasks/data_loaders/arxiv21.py -------------------------------------------------------------------------------- /tasks/data_loaders/cola.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/tasks/data_loaders/cola.py -------------------------------------------------------------------------------- /tasks/data_loaders/qnli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/tasks/data_loaders/qnli.py -------------------------------------------------------------------------------- /tasks/data_loaders/wikitext.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/tasks/data_loaders/wikitext.py -------------------------------------------------------------------------------- /trace_json/__dummy_file__: -------------------------------------------------------------------------------- 1 | make sure the folder is not empty 2 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/dist_args_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/utils/dist_args_utils.py -------------------------------------------------------------------------------- /utils/dist_debug_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/utils/dist_debug_utils.py -------------------------------------------------------------------------------- /utils/dist_test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/utils/dist_test_utils.py -------------------------------------------------------------------------------- /utils/dist_train_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DS3Lab/AC-SGD/HEAD/utils/dist_train_utils.py --------------------------------------------------------------------------------