├── .gitignore ├── LICENSE ├── README.md ├── eval_infbench.py ├── eval_infbench.sh ├── eval_longbench.sh ├── eval_passkey.py ├── eval_passkey.sh ├── infbench_src ├── args.py ├── compute_scores.py ├── eval_chatglm.py ├── eval_gpt4.py ├── eval_utils.py ├── eval_yarn_mistral.py ├── eval_yi_200k.py ├── prompt.py └── yarn_mistral │ ├── configuration_mistral.py │ └── modeling_mistral_yarn.py ├── main ├── download.py ├── eval_lm.py ├── eval_longbench.py ├── eval_longeval.py ├── longbench_utils.py └── train.py ├── src ├── __init__.py ├── activation_beacon_llama │ ├── __init__.py │ ├── configuration_llama.py │ ├── modeling_beacon.py │ ├── modeling_llama.py │ └── modeling_retrieval.py ├── args.py ├── colossal.py ├── data.py ├── deepspeed │ ├── stage2.json │ └── stage2_offload.json ├── metrics.py ├── trainer.py └── util.py └── train.sh /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/README.md -------------------------------------------------------------------------------- /eval_infbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/eval_infbench.py -------------------------------------------------------------------------------- /eval_infbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/eval_infbench.sh -------------------------------------------------------------------------------- /eval_longbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/eval_longbench.sh -------------------------------------------------------------------------------- /eval_passkey.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/eval_passkey.py -------------------------------------------------------------------------------- /eval_passkey.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/eval_passkey.sh -------------------------------------------------------------------------------- /infbench_src/args.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/infbench_src/args.py -------------------------------------------------------------------------------- /infbench_src/compute_scores.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/infbench_src/compute_scores.py -------------------------------------------------------------------------------- /infbench_src/eval_chatglm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/infbench_src/eval_chatglm.py -------------------------------------------------------------------------------- /infbench_src/eval_gpt4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/infbench_src/eval_gpt4.py -------------------------------------------------------------------------------- /infbench_src/eval_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/infbench_src/eval_utils.py -------------------------------------------------------------------------------- /infbench_src/eval_yarn_mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/infbench_src/eval_yarn_mistral.py -------------------------------------------------------------------------------- /infbench_src/eval_yi_200k.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/infbench_src/eval_yi_200k.py -------------------------------------------------------------------------------- /infbench_src/prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/infbench_src/prompt.py -------------------------------------------------------------------------------- /infbench_src/yarn_mistral/configuration_mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/infbench_src/yarn_mistral/configuration_mistral.py -------------------------------------------------------------------------------- /infbench_src/yarn_mistral/modeling_mistral_yarn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/infbench_src/yarn_mistral/modeling_mistral_yarn.py -------------------------------------------------------------------------------- /main/download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/main/download.py -------------------------------------------------------------------------------- /main/eval_lm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/main/eval_lm.py -------------------------------------------------------------------------------- /main/eval_longbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/main/eval_longbench.py -------------------------------------------------------------------------------- /main/eval_longeval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/main/eval_longeval.py -------------------------------------------------------------------------------- /main/longbench_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/main/longbench_utils.py -------------------------------------------------------------------------------- /main/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/main/train.py -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/__init__.py -------------------------------------------------------------------------------- /src/activation_beacon_llama/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/activation_beacon_llama/__init__.py -------------------------------------------------------------------------------- /src/activation_beacon_llama/configuration_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/activation_beacon_llama/configuration_llama.py -------------------------------------------------------------------------------- /src/activation_beacon_llama/modeling_beacon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/activation_beacon_llama/modeling_beacon.py -------------------------------------------------------------------------------- /src/activation_beacon_llama/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/activation_beacon_llama/modeling_llama.py -------------------------------------------------------------------------------- /src/activation_beacon_llama/modeling_retrieval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/activation_beacon_llama/modeling_retrieval.py -------------------------------------------------------------------------------- /src/args.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/args.py -------------------------------------------------------------------------------- /src/colossal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/colossal.py -------------------------------------------------------------------------------- /src/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/data.py -------------------------------------------------------------------------------- /src/deepspeed/stage2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/deepspeed/stage2.json -------------------------------------------------------------------------------- /src/deepspeed/stage2_offload.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/deepspeed/stage2_offload.json -------------------------------------------------------------------------------- /src/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/metrics.py -------------------------------------------------------------------------------- /src/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/trainer.py -------------------------------------------------------------------------------- /src/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/src/util.py -------------------------------------------------------------------------------- /train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leezythu/FocusLLM/HEAD/train.sh --------------------------------------------------------------------------------