├── .flake8 ├── .gitignore ├── LICENSE ├── README.md ├── assets └── figures │ ├── arc-agi.png │ ├── method.png │ └── teaser.png ├── configs ├── sparsity │ ├── llama2-13b-csr.yaml │ ├── llama2-13b-dense.yaml │ ├── llama2-13b-math10k.yaml │ ├── llama2-7b-codefeedback.yaml │ ├── llama2-7b-csr.yaml │ ├── llama2-7b-dense.yaml │ ├── llama2-7b-math10k.yaml │ ├── llama3-8b-csr.yaml │ ├── llama3-8b-dense.yaml │ ├── llama3-8b-glue.yaml │ ├── llama3-8b-math10k.yaml │ ├── llama3.1-8b-arc-agi.yaml │ ├── llama3.1-8b-codefeedback.yaml │ └── llama3.1-8b-wizardlm.yaml └── train │ ├── arc_agi_train.yaml │ ├── codefeedback_train.yaml │ ├── csr170k_train.yaml │ ├── glue_train.yaml │ ├── math10k_train.yaml │ └── wizardlm_train.yaml ├── pyproject.toml ├── scripts ├── arc_agi.sh ├── codefeedback.sh ├── csr170k.sh ├── glue.sh ├── math10k.sh ├── setup │ ├── data.sh │ ├── environment_setup.sh │ ├── setup.sh │ └── svd_estimator.sh ├── speedup.sh └── wizardlm.sh ├── spft ├── api.py ├── callbacks.py ├── data.py ├── modules │ ├── __init__.py │ ├── attn.py │ ├── base.py │ ├── linear.py │ ├── linear4bit.py │ ├── mapping.py │ ├── mlp.py │ ├── pred.py │ ├── svd.py │ └── unsloth │ │ ├── __init__.py │ │ ├── attn.py │ │ ├── lora.py │ │ └── mlp.py ├── test │ ├── arc_agi │ │ ├── arc_agi.py │ │ ├── eval_transduction_api.py │ │ └── validation_transduction_prompt.jsonl │ ├── code.py │ └── main.py ├── train │ ├── args.py │ ├── main.py │ ├── run_glue.py │ └── speedup.py └── utils │ ├── __init__.py │ ├── data.py │ ├── distributed.py │ ├── io.py │ ├── model.py │ ├── module.py │ └── processor.py └── tools ├── analyze.py ├── merge.py ├── mt-bench.py ├── svd.py └── utils └── timer_util.py /.flake8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/.flake8 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/README.md -------------------------------------------------------------------------------- /assets/figures/arc-agi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/assets/figures/arc-agi.png -------------------------------------------------------------------------------- /assets/figures/method.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/assets/figures/method.png -------------------------------------------------------------------------------- /assets/figures/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/assets/figures/teaser.png -------------------------------------------------------------------------------- /configs/sparsity/llama2-13b-csr.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama2-13b-csr.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama2-13b-dense.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama2-13b-dense.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama2-13b-math10k.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama2-13b-math10k.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama2-7b-codefeedback.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama2-7b-codefeedback.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama2-7b-csr.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama2-7b-csr.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama2-7b-dense.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama2-7b-dense.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama2-7b-math10k.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama2-7b-math10k.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama3-8b-csr.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama3-8b-csr.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama3-8b-dense.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama3-8b-dense.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama3-8b-glue.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama3-8b-glue.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama3-8b-math10k.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama3-8b-math10k.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama3.1-8b-arc-agi.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama3.1-8b-arc-agi.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama3.1-8b-codefeedback.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama3.1-8b-codefeedback.yaml -------------------------------------------------------------------------------- /configs/sparsity/llama3.1-8b-wizardlm.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/sparsity/llama3.1-8b-wizardlm.yaml -------------------------------------------------------------------------------- /configs/train/arc_agi_train.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/train/arc_agi_train.yaml -------------------------------------------------------------------------------- /configs/train/codefeedback_train.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/train/codefeedback_train.yaml -------------------------------------------------------------------------------- /configs/train/csr170k_train.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/train/csr170k_train.yaml -------------------------------------------------------------------------------- /configs/train/glue_train.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/train/glue_train.yaml -------------------------------------------------------------------------------- /configs/train/math10k_train.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/train/math10k_train.yaml -------------------------------------------------------------------------------- /configs/train/wizardlm_train.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/configs/train/wizardlm_train.yaml -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/arc_agi.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/scripts/arc_agi.sh -------------------------------------------------------------------------------- /scripts/codefeedback.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/scripts/codefeedback.sh -------------------------------------------------------------------------------- /scripts/csr170k.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/scripts/csr170k.sh -------------------------------------------------------------------------------- /scripts/glue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/scripts/glue.sh -------------------------------------------------------------------------------- /scripts/math10k.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/scripts/math10k.sh -------------------------------------------------------------------------------- /scripts/setup/data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/scripts/setup/data.sh -------------------------------------------------------------------------------- /scripts/setup/environment_setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/scripts/setup/environment_setup.sh -------------------------------------------------------------------------------- /scripts/setup/setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/scripts/setup/setup.sh -------------------------------------------------------------------------------- /scripts/setup/svd_estimator.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/scripts/setup/svd_estimator.sh -------------------------------------------------------------------------------- /scripts/speedup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/scripts/speedup.sh -------------------------------------------------------------------------------- /scripts/wizardlm.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/scripts/wizardlm.sh -------------------------------------------------------------------------------- /spft/api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/api.py -------------------------------------------------------------------------------- /spft/callbacks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/callbacks.py -------------------------------------------------------------------------------- /spft/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/data.py -------------------------------------------------------------------------------- /spft/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/__init__.py -------------------------------------------------------------------------------- /spft/modules/attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/attn.py -------------------------------------------------------------------------------- /spft/modules/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/base.py -------------------------------------------------------------------------------- /spft/modules/linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/linear.py -------------------------------------------------------------------------------- /spft/modules/linear4bit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/linear4bit.py -------------------------------------------------------------------------------- /spft/modules/mapping.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/mapping.py -------------------------------------------------------------------------------- /spft/modules/mlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/mlp.py -------------------------------------------------------------------------------- /spft/modules/pred.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/pred.py -------------------------------------------------------------------------------- /spft/modules/svd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/svd.py -------------------------------------------------------------------------------- /spft/modules/unsloth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/unsloth/__init__.py -------------------------------------------------------------------------------- /spft/modules/unsloth/attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/unsloth/attn.py -------------------------------------------------------------------------------- /spft/modules/unsloth/lora.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/unsloth/lora.py -------------------------------------------------------------------------------- /spft/modules/unsloth/mlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/modules/unsloth/mlp.py -------------------------------------------------------------------------------- /spft/test/arc_agi/arc_agi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/test/arc_agi/arc_agi.py -------------------------------------------------------------------------------- /spft/test/arc_agi/eval_transduction_api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/test/arc_agi/eval_transduction_api.py -------------------------------------------------------------------------------- /spft/test/arc_agi/validation_transduction_prompt.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/test/arc_agi/validation_transduction_prompt.jsonl -------------------------------------------------------------------------------- /spft/test/code.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/test/code.py -------------------------------------------------------------------------------- /spft/test/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/test/main.py -------------------------------------------------------------------------------- /spft/train/args.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/train/args.py -------------------------------------------------------------------------------- /spft/train/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/train/main.py -------------------------------------------------------------------------------- /spft/train/run_glue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/train/run_glue.py -------------------------------------------------------------------------------- /spft/train/speedup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/train/speedup.py -------------------------------------------------------------------------------- /spft/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .module import * 2 | -------------------------------------------------------------------------------- /spft/utils/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/utils/data.py -------------------------------------------------------------------------------- /spft/utils/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/utils/distributed.py -------------------------------------------------------------------------------- /spft/utils/io.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/utils/io.py -------------------------------------------------------------------------------- /spft/utils/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/utils/model.py -------------------------------------------------------------------------------- /spft/utils/module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/utils/module.py -------------------------------------------------------------------------------- /spft/utils/processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/spft/utils/processor.py -------------------------------------------------------------------------------- /tools/analyze.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/tools/analyze.py -------------------------------------------------------------------------------- /tools/merge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/tools/merge.py -------------------------------------------------------------------------------- /tools/mt-bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/tools/mt-bench.py -------------------------------------------------------------------------------- /tools/svd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/tools/svd.py -------------------------------------------------------------------------------- /tools/utils/timer_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/z-lab/sparselora/HEAD/tools/utils/timer_util.py --------------------------------------------------------------------------------