├── LICENSE ├── MEAP-Pretain ├── c4_bin │ └── c4_0_0000000006.bin ├── convert │ ├── convert_lit_checkpoint.py │ └── convert_safetensors.py ├── data_process │ ├── gz_unzip_v1.py │ └── prepare_c4.py ├── lit_gpt │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── config.cpython-310.pyc │ │ ├── fused_cross_entropy.cpython-310.pyc │ │ ├── fused_rotary_embedding.cpython-310.pyc │ │ ├── model.cpython-310.pyc │ │ ├── packed_dataset.cpython-310.pyc │ │ ├── rmsnorm.cpython-310.pyc │ │ ├── speed_monitor.cpython-310.pyc │ │ ├── tokenizer.cpython-310.pyc │ │ └── utils.cpython-310.pyc │ ├── adapter.py │ ├── adapter_v2.py │ ├── config.py │ ├── fused_cross_entropy.py │ ├── fused_rotary_embedding.py │ ├── lora.py │ ├── model.py │ ├── packed_dataset.py │ ├── rmsnorm.py │ ├── speed_monitor.py │ ├── tokenizer.py │ └── utils.py ├── pretrained │ ├── meap_0.1b_2b.py │ ├── meap_0.1b_2b_0.05.py │ ├── meap_0.1b_2b_0.1.py │ ├── meap_0.3b_5b.py │ ├── meap_0.3b_5b_0.05.py │ ├── meap_0.3b_5b_0.1.py │ ├── meap_0.3b_5b_0.2.py │ ├── meap_0.5b_10b.py │ ├── meap_0.5b_10b_0.05.py │ ├── meap_0.5b_10b_0.1.py │ ├── meap_0.5b_10b_0.2.py │ ├── meap_1b.py │ ├── meap_1b_0.05.py │ ├── meap_1b_0.1.py │ ├── meap_1b_0.15_20b.py │ ├── meap_1b_0.15_40b.py │ ├── meap_1b_0.15_60b.py │ └── meap_1b_0.2.py ├── requirements.txt ├── run │ └── run_one_node.sh └── tokenizer │ ├── special_tokens_map.json │ ├── tokenizer.json │ ├── tokenizer.model │ └── tokenizer_config.json ├── MEAP-SFT ├── MEAP-SFT.py ├── deepspeed_zero_stage2_config.json ├── requirements.txt └── template.py ├── README.md └── data └── alpaca_data.jsonl /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/LICENSE -------------------------------------------------------------------------------- /MEAP-Pretain/c4_bin/c4_0_0000000006.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/c4_bin/c4_0_0000000006.bin -------------------------------------------------------------------------------- /MEAP-Pretain/convert/convert_lit_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/convert/convert_lit_checkpoint.py -------------------------------------------------------------------------------- /MEAP-Pretain/convert/convert_safetensors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/convert/convert_safetensors.py -------------------------------------------------------------------------------- /MEAP-Pretain/data_process/gz_unzip_v1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/data_process/gz_unzip_v1.py -------------------------------------------------------------------------------- /MEAP-Pretain/data_process/prepare_c4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/data_process/prepare_c4.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/__init__.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/__pycache__/config.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/__pycache__/config.cpython-310.pyc -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/__pycache__/fused_cross_entropy.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/__pycache__/fused_cross_entropy.cpython-310.pyc -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/__pycache__/fused_rotary_embedding.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/__pycache__/fused_rotary_embedding.cpython-310.pyc -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/__pycache__/model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/__pycache__/model.cpython-310.pyc -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/__pycache__/packed_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/__pycache__/packed_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/__pycache__/rmsnorm.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/__pycache__/rmsnorm.cpython-310.pyc -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/__pycache__/speed_monitor.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/__pycache__/speed_monitor.cpython-310.pyc -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/__pycache__/tokenizer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/__pycache__/tokenizer.cpython-310.pyc -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/__pycache__/utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/__pycache__/utils.cpython-310.pyc -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/adapter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/adapter.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/adapter_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/adapter_v2.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/config.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/fused_cross_entropy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/fused_cross_entropy.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/fused_rotary_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/fused_rotary_embedding.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/lora.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/lora.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/model.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/packed_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/packed_dataset.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/rmsnorm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/rmsnorm.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/speed_monitor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/speed_monitor.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/tokenizer.py -------------------------------------------------------------------------------- /MEAP-Pretain/lit_gpt/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/lit_gpt/utils.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_0.1b_2b.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_0.1b_2b.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_0.1b_2b_0.05.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_0.1b_2b_0.05.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_0.1b_2b_0.1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_0.1b_2b_0.1.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_0.3b_5b.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_0.3b_5b.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_0.3b_5b_0.05.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_0.3b_5b_0.05.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_0.3b_5b_0.1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_0.3b_5b_0.1.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_0.3b_5b_0.2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_0.3b_5b_0.2.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_0.5b_10b.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_0.5b_10b.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_0.5b_10b_0.05.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_0.5b_10b_0.05.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_0.5b_10b_0.1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_0.5b_10b_0.1.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_0.5b_10b_0.2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_0.5b_10b_0.2.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_1b.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_1b.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_1b_0.05.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_1b_0.05.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_1b_0.1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_1b_0.1.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_1b_0.15_20b.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_1b_0.15_20b.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_1b_0.15_40b.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_1b_0.15_40b.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_1b_0.15_60b.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_1b_0.15_60b.py -------------------------------------------------------------------------------- /MEAP-Pretain/pretrained/meap_1b_0.2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/pretrained/meap_1b_0.2.py -------------------------------------------------------------------------------- /MEAP-Pretain/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/requirements.txt -------------------------------------------------------------------------------- /MEAP-Pretain/run/run_one_node.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/run/run_one_node.sh -------------------------------------------------------------------------------- /MEAP-Pretain/tokenizer/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/tokenizer/special_tokens_map.json -------------------------------------------------------------------------------- /MEAP-Pretain/tokenizer/tokenizer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/tokenizer/tokenizer.json -------------------------------------------------------------------------------- /MEAP-Pretain/tokenizer/tokenizer.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/tokenizer/tokenizer.model -------------------------------------------------------------------------------- /MEAP-Pretain/tokenizer/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-Pretain/tokenizer/tokenizer_config.json -------------------------------------------------------------------------------- /MEAP-SFT/MEAP-SFT.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-SFT/MEAP-SFT.py -------------------------------------------------------------------------------- /MEAP-SFT/deepspeed_zero_stage2_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-SFT/deepspeed_zero_stage2_config.json -------------------------------------------------------------------------------- /MEAP-SFT/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-SFT/requirements.txt -------------------------------------------------------------------------------- /MEAP-SFT/template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/MEAP-SFT/template.py -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/README.md -------------------------------------------------------------------------------- /data/alpaca_data.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scitix/MEAP/HEAD/data/alpaca_data.jsonl --------------------------------------------------------------------------------