├── .gitignore ├── .pre-commit-config.yaml ├── README.md ├── decoder ├── __init__.py ├── odysseus.py ├── ring.py ├── tensor_parallel.py └── ulysses.py ├── media ├── Odysseus.jpg ├── ody_perf_2.png └── odysseus_perf.png ├── requirements.txt ├── run.sh ├── tests ├── attention_test.py └── mlp_test.py ├── train.py └── utils ├── allgather.py ├── apply_seq_parallel.py ├── comm.py ├── globals.py ├── linear_tpsp.py ├── linear_zero3.py └── log_utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/* 2 | *pyc 3 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/README.md -------------------------------------------------------------------------------- /decoder/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /decoder/odysseus.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/decoder/odysseus.py -------------------------------------------------------------------------------- /decoder/ring.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/decoder/ring.py -------------------------------------------------------------------------------- /decoder/tensor_parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/decoder/tensor_parallel.py -------------------------------------------------------------------------------- /decoder/ulysses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/decoder/ulysses.py -------------------------------------------------------------------------------- /media/Odysseus.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/media/Odysseus.jpg -------------------------------------------------------------------------------- /media/ody_perf_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/media/ody_perf_2.png -------------------------------------------------------------------------------- /media/odysseus_perf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/media/odysseus_perf.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/requirements.txt -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/run.sh -------------------------------------------------------------------------------- /tests/attention_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/tests/attention_test.py -------------------------------------------------------------------------------- /tests/mlp_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/tests/mlp_test.py -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/train.py -------------------------------------------------------------------------------- /utils/allgather.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/utils/allgather.py -------------------------------------------------------------------------------- /utils/apply_seq_parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/utils/apply_seq_parallel.py -------------------------------------------------------------------------------- /utils/comm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/utils/comm.py -------------------------------------------------------------------------------- /utils/globals.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/utils/globals.py -------------------------------------------------------------------------------- /utils/linear_tpsp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/utils/linear_tpsp.py -------------------------------------------------------------------------------- /utils/linear_zero3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/utils/linear_zero3.py -------------------------------------------------------------------------------- /utils/log_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feifeibear/Odysseus-Transformer/HEAD/utils/log_utils.py --------------------------------------------------------------------------------