├── .gitignore ├── LICENSE ├── README.md ├── ckpt ├── convert.py ├── stage1 │ └── router.pth └── stage2 │ ├── lora.pth │ └── router.pth ├── custom_trainer.py ├── dataset_processing.py ├── deepspeed_config.json ├── image └── introduction.png ├── inference.py ├── inference ├── README.md ├── __init__.py ├── benchmark.py ├── ops │ ├── __init__.py │ ├── attn │ │ ├── __init__.py │ │ ├── sparse_attn.py │ │ └── sparse_rope.py │ ├── mlp │ │ ├── __init__.py │ │ ├── glu.py │ │ └── sparse_mlp.py │ ├── sparse_rmsnorm.py │ └── utils.py └── wrapper │ ├── __init__.py │ ├── base.py │ └── model │ ├── __init__.py │ ├── base.py │ ├── cache.py │ └── llama.py ├── log_sparsity.py ├── main.py ├── models.py ├── requirements.txt ├── router_attn_mlp.py ├── torch_save.py └── train_processing_two_stages.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.pyc 3 | .DS_Store -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/README.md -------------------------------------------------------------------------------- /ckpt/convert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/ckpt/convert.py -------------------------------------------------------------------------------- /ckpt/stage1/router.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/ckpt/stage1/router.pth -------------------------------------------------------------------------------- /ckpt/stage2/lora.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/ckpt/stage2/lora.pth -------------------------------------------------------------------------------- /ckpt/stage2/router.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/ckpt/stage2/router.pth -------------------------------------------------------------------------------- /custom_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/custom_trainer.py -------------------------------------------------------------------------------- /dataset_processing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/dataset_processing.py -------------------------------------------------------------------------------- /deepspeed_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/deepspeed_config.json -------------------------------------------------------------------------------- /image/introduction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/image/introduction.png -------------------------------------------------------------------------------- /inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference.py -------------------------------------------------------------------------------- /inference/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/README.md -------------------------------------------------------------------------------- /inference/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /inference/benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/benchmark.py -------------------------------------------------------------------------------- /inference/ops/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/ops/__init__.py -------------------------------------------------------------------------------- /inference/ops/attn/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /inference/ops/attn/sparse_attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/ops/attn/sparse_attn.py -------------------------------------------------------------------------------- /inference/ops/attn/sparse_rope.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/ops/attn/sparse_rope.py -------------------------------------------------------------------------------- /inference/ops/mlp/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /inference/ops/mlp/glu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/ops/mlp/glu.py -------------------------------------------------------------------------------- /inference/ops/mlp/sparse_mlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/ops/mlp/sparse_mlp.py -------------------------------------------------------------------------------- /inference/ops/sparse_rmsnorm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/ops/sparse_rmsnorm.py -------------------------------------------------------------------------------- /inference/ops/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/ops/utils.py -------------------------------------------------------------------------------- /inference/wrapper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/wrapper/__init__.py -------------------------------------------------------------------------------- /inference/wrapper/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/wrapper/base.py -------------------------------------------------------------------------------- /inference/wrapper/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/wrapper/model/__init__.py -------------------------------------------------------------------------------- /inference/wrapper/model/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/wrapper/model/base.py -------------------------------------------------------------------------------- /inference/wrapper/model/cache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/wrapper/model/cache.py -------------------------------------------------------------------------------- /inference/wrapper/model/llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/inference/wrapper/model/llama.py -------------------------------------------------------------------------------- /log_sparsity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/log_sparsity.py -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/main.py -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/models.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/requirements.txt -------------------------------------------------------------------------------- /router_attn_mlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/router_attn_mlp.py -------------------------------------------------------------------------------- /torch_save.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/torch_save.py -------------------------------------------------------------------------------- /train_processing_two_stages.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EIT-NLP/SkipGPT/HEAD/train_processing_two_stages.py --------------------------------------------------------------------------------