├── .gitignore ├── README.md ├── __init__.py ├── asset ├── MemLong.png └── illustration.png ├── data ├── process.sh └── text_processing.py ├── eval ├── icl │ ├── MemLong │ │ ├── __init__.py │ │ ├── align_memory.py │ │ ├── cache_utils.py │ │ ├── configuration_llama.py │ │ ├── modeling_llama.py │ │ ├── modeling_llama_position.py │ │ ├── ret_embedder.py │ │ ├── toolkit.py │ │ └── utils.py │ ├── data │ │ ├── SST-2 │ │ │ ├── dev.tsv │ │ │ └── train.tsv │ │ ├── mr │ │ │ └── train.csv │ │ ├── sst-5 │ │ │ └── train.csv │ │ └── subj │ │ │ └── train.csv │ ├── eval_icl.py │ └── longllama │ │ ├── __init__.py │ │ ├── configuration_longllama.py │ │ ├── longllama_utils.py │ │ └── modeling_longllama.py └── language_modeling │ ├── CEPE │ ├── __init__.py │ └── modeling_llama_flash.py │ ├── InfiniteTransformer │ ├── __init__.py │ ├── configuration_infini_gemma.py │ └── modeling_infini_gemma.py │ ├── LongLora │ ├── __init__.py │ └── llama_attn_replace.py │ ├── MemLong │ ├── __init__.py │ ├── align_memory.py │ ├── cache_utils.py │ ├── configuration_llama.py │ ├── modeling_llama.py │ ├── modeling_llama_position.py │ ├── ret_embedder.py │ ├── toolkit.py │ └── utils.py │ ├── eval.py │ ├── get_data.py │ ├── longllama │ ├── __init__.py │ ├── configuration_longllama.py │ ├── longllama_utils.py │ └── modeling_longllama.py │ ├── script │ ├── eval_CEPE.sh │ ├── eval_gemma.sh │ ├── eval_llama.sh │ ├── eval_longllama.sh │ ├── eval_longlora.sh │ ├── eval_memlong.sh │ ├── eval_mistral.sh │ ├── eval_openllama.sh │ ├── eval_phi.sh │ └── eval_yarn.sh │ └── utils.py ├── instruction_finetuning ├── __init__.py ├── arguments.py ├── config_file.yaml ├── data_processing.py ├── supervised_finetuning.py └── utils.py ├── requirements.txt ├── run_clm_no_trainer.py ├── src ├── __init__.py ├── align_memory.py ├── cache_utils.py ├── configuration_llama.py ├── modeling_llama_position.py ├── ret_embedder.py ├── toolkit.py └── utils.py ├── train_stage1.sh ├── train_stage2.sh ├── zero2_config.yaml └── zero_stage2_config.json /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/README.md -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /asset/MemLong.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/asset/MemLong.png -------------------------------------------------------------------------------- /asset/illustration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/asset/illustration.png -------------------------------------------------------------------------------- /data/process.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/data/process.sh -------------------------------------------------------------------------------- /data/text_processing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/data/text_processing.py -------------------------------------------------------------------------------- /eval/icl/MemLong/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/icl/MemLong/align_memory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/MemLong/align_memory.py -------------------------------------------------------------------------------- /eval/icl/MemLong/cache_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/MemLong/cache_utils.py -------------------------------------------------------------------------------- /eval/icl/MemLong/configuration_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/MemLong/configuration_llama.py -------------------------------------------------------------------------------- /eval/icl/MemLong/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/MemLong/modeling_llama.py -------------------------------------------------------------------------------- /eval/icl/MemLong/modeling_llama_position.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/MemLong/modeling_llama_position.py -------------------------------------------------------------------------------- /eval/icl/MemLong/ret_embedder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/MemLong/ret_embedder.py -------------------------------------------------------------------------------- /eval/icl/MemLong/toolkit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/MemLong/toolkit.py -------------------------------------------------------------------------------- /eval/icl/MemLong/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/MemLong/utils.py -------------------------------------------------------------------------------- /eval/icl/data/SST-2/dev.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/data/SST-2/dev.tsv -------------------------------------------------------------------------------- /eval/icl/data/SST-2/train.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/data/SST-2/train.tsv -------------------------------------------------------------------------------- /eval/icl/data/mr/train.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/data/mr/train.csv -------------------------------------------------------------------------------- /eval/icl/data/sst-5/train.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/data/sst-5/train.csv -------------------------------------------------------------------------------- /eval/icl/data/subj/train.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/data/subj/train.csv -------------------------------------------------------------------------------- /eval/icl/eval_icl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/eval_icl.py -------------------------------------------------------------------------------- /eval/icl/longllama/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/icl/longllama/configuration_longllama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/longllama/configuration_longllama.py -------------------------------------------------------------------------------- /eval/icl/longllama/longllama_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/longllama/longllama_utils.py -------------------------------------------------------------------------------- /eval/icl/longllama/modeling_longllama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/icl/longllama/modeling_longllama.py -------------------------------------------------------------------------------- /eval/language_modeling/CEPE/__init__.py: -------------------------------------------------------------------------------- 1 | from .modeling_llama_flash import LlamaForCausalContextLM -------------------------------------------------------------------------------- /eval/language_modeling/CEPE/modeling_llama_flash.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/CEPE/modeling_llama_flash.py -------------------------------------------------------------------------------- /eval/language_modeling/InfiniteTransformer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/InfiniteTransformer/__init__.py -------------------------------------------------------------------------------- /eval/language_modeling/InfiniteTransformer/configuration_infini_gemma.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/InfiniteTransformer/configuration_infini_gemma.py -------------------------------------------------------------------------------- /eval/language_modeling/InfiniteTransformer/modeling_infini_gemma.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/InfiniteTransformer/modeling_infini_gemma.py -------------------------------------------------------------------------------- /eval/language_modeling/LongLora/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/LongLora/__init__.py -------------------------------------------------------------------------------- /eval/language_modeling/LongLora/llama_attn_replace.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/LongLora/llama_attn_replace.py -------------------------------------------------------------------------------- /eval/language_modeling/MemLong/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/language_modeling/MemLong/align_memory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/MemLong/align_memory.py -------------------------------------------------------------------------------- /eval/language_modeling/MemLong/cache_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/MemLong/cache_utils.py -------------------------------------------------------------------------------- /eval/language_modeling/MemLong/configuration_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/MemLong/configuration_llama.py -------------------------------------------------------------------------------- /eval/language_modeling/MemLong/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/MemLong/modeling_llama.py -------------------------------------------------------------------------------- /eval/language_modeling/MemLong/modeling_llama_position.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/MemLong/modeling_llama_position.py -------------------------------------------------------------------------------- /eval/language_modeling/MemLong/ret_embedder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/MemLong/ret_embedder.py -------------------------------------------------------------------------------- /eval/language_modeling/MemLong/toolkit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/MemLong/toolkit.py -------------------------------------------------------------------------------- /eval/language_modeling/MemLong/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/MemLong/utils.py -------------------------------------------------------------------------------- /eval/language_modeling/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/eval.py -------------------------------------------------------------------------------- /eval/language_modeling/get_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/get_data.py -------------------------------------------------------------------------------- /eval/language_modeling/longllama/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/language_modeling/longllama/configuration_longllama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/longllama/configuration_longllama.py -------------------------------------------------------------------------------- /eval/language_modeling/longllama/longllama_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/longllama/longllama_utils.py -------------------------------------------------------------------------------- /eval/language_modeling/longllama/modeling_longllama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/longllama/modeling_longllama.py -------------------------------------------------------------------------------- /eval/language_modeling/script/eval_CEPE.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/script/eval_CEPE.sh -------------------------------------------------------------------------------- /eval/language_modeling/script/eval_gemma.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/script/eval_gemma.sh -------------------------------------------------------------------------------- /eval/language_modeling/script/eval_llama.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/script/eval_llama.sh -------------------------------------------------------------------------------- /eval/language_modeling/script/eval_longllama.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/script/eval_longllama.sh -------------------------------------------------------------------------------- /eval/language_modeling/script/eval_longlora.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/script/eval_longlora.sh -------------------------------------------------------------------------------- /eval/language_modeling/script/eval_memlong.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/script/eval_memlong.sh -------------------------------------------------------------------------------- /eval/language_modeling/script/eval_mistral.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/script/eval_mistral.sh -------------------------------------------------------------------------------- /eval/language_modeling/script/eval_openllama.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/script/eval_openllama.sh -------------------------------------------------------------------------------- /eval/language_modeling/script/eval_phi.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/script/eval_phi.sh -------------------------------------------------------------------------------- /eval/language_modeling/script/eval_yarn.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/script/eval_yarn.sh -------------------------------------------------------------------------------- /eval/language_modeling/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/eval/language_modeling/utils.py -------------------------------------------------------------------------------- /instruction_finetuning/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /instruction_finetuning/arguments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/instruction_finetuning/arguments.py -------------------------------------------------------------------------------- /instruction_finetuning/config_file.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/instruction_finetuning/config_file.yaml -------------------------------------------------------------------------------- /instruction_finetuning/data_processing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/instruction_finetuning/data_processing.py -------------------------------------------------------------------------------- /instruction_finetuning/supervised_finetuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/instruction_finetuning/supervised_finetuning.py -------------------------------------------------------------------------------- /instruction_finetuning/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/instruction_finetuning/utils.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/requirements.txt -------------------------------------------------------------------------------- /run_clm_no_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/run_clm_no_trainer.py -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/align_memory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/src/align_memory.py -------------------------------------------------------------------------------- /src/cache_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/src/cache_utils.py -------------------------------------------------------------------------------- /src/configuration_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/src/configuration_llama.py -------------------------------------------------------------------------------- /src/modeling_llama_position.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/src/modeling_llama_position.py -------------------------------------------------------------------------------- /src/ret_embedder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/src/ret_embedder.py -------------------------------------------------------------------------------- /src/toolkit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/src/toolkit.py -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/src/utils.py -------------------------------------------------------------------------------- /train_stage1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/train_stage1.sh -------------------------------------------------------------------------------- /train_stage2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/train_stage2.sh -------------------------------------------------------------------------------- /zero2_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/zero2_config.yaml -------------------------------------------------------------------------------- /zero_stage2_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bui1dMySea/MemLong/HEAD/zero_stage2_config.json --------------------------------------------------------------------------------