├── .gitignore ├── LICENSE ├── PaulGrahamEssays ├── addiction.txt ├── aord.txt ├── apple.txt ├── avg.txt ├── before.txt ├── bias.txt ├── boss.txt ├── copy.txt ├── corpdev.txt ├── desres.txt ├── diff.txt ├── ecw.txt ├── founders.txt ├── foundervisa.txt ├── gap.txt ├── gba.txt ├── gh.txt ├── goodtaste.txt ├── hubs.txt ├── iflisp.txt ├── island.txt ├── know.txt ├── langdes.txt ├── laundry.txt ├── love.txt ├── mod.txt ├── newideas.txt ├── nft.txt ├── philosophy.txt ├── popular.txt ├── pow.txt ├── rootsoflisp.txt ├── rss.txt ├── siliconvalley.txt ├── startuplessons.txt ├── submarine.txt ├── sun.txt ├── superangels.txt ├── todo.txt ├── unions.txt ├── useful.txt ├── vb.txt ├── vcsqueeze.txt ├── vw.txt ├── want.txt ├── web20.txt ├── weird.txt ├── wisdom.txt └── worked.txt ├── README.md ├── accelerate_configs ├── deepspeed_inference.yaml ├── single_node.yaml ├── two_node.yaml ├── zero3_offload.json └── zero3_offload_inference.json ├── data ├── Logo.webp ├── needle.jpg └── ppl.png ├── easy_context ├── __init__.py ├── dist_flash_attn │ ├── README.md │ ├── async_communication.py │ ├── lightseq_async_attn.py │ ├── lightseq_async_attn_varlen.py │ ├── monkey_patch.py │ └── prepare_input.py ├── ulysses_attn │ ├── monkey_patch.py │ └── prepare_inputs.py ├── unsloth_offloaded_gradient_checkpoint │ └── monkey_patch.py ├── usp │ ├── monkey_patch.py │ └── prepare_inputs.py └── zigzag_ring_attn │ ├── monkey_patch.py │ └── prepare_inputs.py ├── eval_needle.py ├── eval_ppl.py ├── plot.py ├── requirements.txt ├── train.py └── train_scripts ├── EasyContext-1M-Llama-2-7B.sh └── EasyContext-256K-danube2.sh /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/LICENSE -------------------------------------------------------------------------------- /PaulGrahamEssays/addiction.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/addiction.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/aord.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/aord.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/apple.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/apple.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/avg.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/avg.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/before.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/before.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/bias.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/bias.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/boss.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/boss.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/copy.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/copy.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/corpdev.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/corpdev.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/desres.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/desres.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/diff.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/diff.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/ecw.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/ecw.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/founders.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/founders.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/foundervisa.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/foundervisa.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/gap.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/gap.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/gba.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/gba.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/gh.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/gh.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/goodtaste.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/goodtaste.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/hubs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/hubs.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/iflisp.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/iflisp.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/island.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/island.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/know.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/know.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/langdes.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/langdes.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/laundry.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/laundry.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/love.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/love.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/mod.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/mod.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/newideas.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/newideas.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/nft.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/nft.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/philosophy.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/philosophy.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/popular.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/popular.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/pow.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/pow.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/rootsoflisp.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/rootsoflisp.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/rss.txt: -------------------------------------------------------------------------------- 1 | Aaron Swartz created a scraped 2 | feed 3 | of the essays page. -------------------------------------------------------------------------------- /PaulGrahamEssays/siliconvalley.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/siliconvalley.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/startuplessons.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/startuplessons.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/submarine.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/submarine.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/sun.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/sun.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/superangels.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/superangels.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/todo.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/todo.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/unions.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/unions.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/useful.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/useful.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/vb.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/vb.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/vcsqueeze.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/vcsqueeze.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/vw.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/vw.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/want.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/want.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/web20.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/web20.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/weird.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/weird.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/wisdom.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/wisdom.txt -------------------------------------------------------------------------------- /PaulGrahamEssays/worked.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/PaulGrahamEssays/worked.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/README.md -------------------------------------------------------------------------------- /accelerate_configs/deepspeed_inference.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/accelerate_configs/deepspeed_inference.yaml -------------------------------------------------------------------------------- /accelerate_configs/single_node.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/accelerate_configs/single_node.yaml -------------------------------------------------------------------------------- /accelerate_configs/two_node.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/accelerate_configs/two_node.yaml -------------------------------------------------------------------------------- /accelerate_configs/zero3_offload.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/accelerate_configs/zero3_offload.json -------------------------------------------------------------------------------- /accelerate_configs/zero3_offload_inference.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/accelerate_configs/zero3_offload_inference.json -------------------------------------------------------------------------------- /data/Logo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/data/Logo.webp -------------------------------------------------------------------------------- /data/needle.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/data/needle.jpg -------------------------------------------------------------------------------- /data/ppl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/data/ppl.png -------------------------------------------------------------------------------- /easy_context/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/__init__.py -------------------------------------------------------------------------------- /easy_context/dist_flash_attn/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/dist_flash_attn/README.md -------------------------------------------------------------------------------- /easy_context/dist_flash_attn/async_communication.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/dist_flash_attn/async_communication.py -------------------------------------------------------------------------------- /easy_context/dist_flash_attn/lightseq_async_attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/dist_flash_attn/lightseq_async_attn.py -------------------------------------------------------------------------------- /easy_context/dist_flash_attn/lightseq_async_attn_varlen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/dist_flash_attn/lightseq_async_attn_varlen.py -------------------------------------------------------------------------------- /easy_context/dist_flash_attn/monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/dist_flash_attn/monkey_patch.py -------------------------------------------------------------------------------- /easy_context/dist_flash_attn/prepare_input.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/dist_flash_attn/prepare_input.py -------------------------------------------------------------------------------- /easy_context/ulysses_attn/monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/ulysses_attn/monkey_patch.py -------------------------------------------------------------------------------- /easy_context/ulysses_attn/prepare_inputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/ulysses_attn/prepare_inputs.py -------------------------------------------------------------------------------- /easy_context/unsloth_offloaded_gradient_checkpoint/monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/unsloth_offloaded_gradient_checkpoint/monkey_patch.py -------------------------------------------------------------------------------- /easy_context/usp/monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/usp/monkey_patch.py -------------------------------------------------------------------------------- /easy_context/usp/prepare_inputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/usp/prepare_inputs.py -------------------------------------------------------------------------------- /easy_context/zigzag_ring_attn/monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/zigzag_ring_attn/monkey_patch.py -------------------------------------------------------------------------------- /easy_context/zigzag_ring_attn/prepare_inputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/easy_context/zigzag_ring_attn/prepare_inputs.py -------------------------------------------------------------------------------- /eval_needle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/eval_needle.py -------------------------------------------------------------------------------- /eval_ppl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/eval_ppl.py -------------------------------------------------------------------------------- /plot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/plot.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/requirements.txt -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/train.py -------------------------------------------------------------------------------- /train_scripts/EasyContext-1M-Llama-2-7B.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/train_scripts/EasyContext-1M-Llama-2-7B.sh -------------------------------------------------------------------------------- /train_scripts/EasyContext-256K-danube2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jzhang38/EasyContext/HEAD/train_scripts/EasyContext-256K-danube2.sh --------------------------------------------------------------------------------