├── .gitignore ├── README.md ├── s01 ├── AfterSessionExercises.txt ├── Session1Slides.pdf ├── in_class_run.py └── run.py ├── s02 ├── AfterSessionExercises.txt ├── Session2Slides.pdf ├── __pycache__ │ └── timing_util.cpython-310.pyc ├── rafi_preclass_prep │ ├── A_addition.py │ ├── B_addition.py │ ├── C_matmul.py │ └── timing_util.py └── timing_util.py ├── s03 ├── AfterSessionExercises.txt ├── Session3Slides.pdf ├── play_with_sharding.py └── play_with_sharding_in_class.py ├── s04 ├── AfterSessionExercises.txt ├── Session4Slides.pdf ├── before_class_allgather.py ├── before_class_matmuls.py ├── in_class_all_gather.py ├── in_class_matmul.py ├── shardings.py └── timing_util.py ├── s05 ├── AfterSessionExercises.txt ├── Session5Slides.pdf ├── before_class_attention.py └── during_class_attention.py ├── s06 ├── AfterSessionExercises.txt ├── Session6Slides.pdf ├── class_end.py ├── class_start.py └── prep.py ├── s07 ├── AfterSessionExercises.txt ├── Session7Slides.pdf ├── class_end.py ├── class_start.py └── prep.py ├── s08 ├── AfterSessionExercises.txt ├── Session8Slides.pdf ├── class_pretend_inference_is_training_prep.py ├── class_pretend_inference_is_training_start.py ├── class_softmax_class.py ├── class_softmax_primer.py ├── class_start.py └── class_training.py ├── s09 ├── AfterSessionExercises.txt ├── Session9Slides.pdf ├── __pycache__ │ └── timing_util.cpython-310.pyc ├── class_real_inference_end.py ├── class_real_inference_prep.py ├── class_real_inference_prep2.py ├── class_real_inference_start.py ├── determinism_end.py ├── determinism_prep.py ├── determinism_start.py └── timing_util.py └── s10 ├── AfterSessionExercises.txt └── Session10Slides.pdf /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/README.md -------------------------------------------------------------------------------- /s01/AfterSessionExercises.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s01/AfterSessionExercises.txt -------------------------------------------------------------------------------- /s01/Session1Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s01/Session1Slides.pdf -------------------------------------------------------------------------------- /s01/in_class_run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s01/in_class_run.py -------------------------------------------------------------------------------- /s01/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s01/run.py -------------------------------------------------------------------------------- /s02/AfterSessionExercises.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s02/AfterSessionExercises.txt -------------------------------------------------------------------------------- /s02/Session2Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s02/Session2Slides.pdf -------------------------------------------------------------------------------- /s02/__pycache__/timing_util.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s02/__pycache__/timing_util.cpython-310.pyc -------------------------------------------------------------------------------- /s02/rafi_preclass_prep/A_addition.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s02/rafi_preclass_prep/A_addition.py -------------------------------------------------------------------------------- /s02/rafi_preclass_prep/B_addition.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s02/rafi_preclass_prep/B_addition.py -------------------------------------------------------------------------------- /s02/rafi_preclass_prep/C_matmul.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s02/rafi_preclass_prep/C_matmul.py -------------------------------------------------------------------------------- /s02/rafi_preclass_prep/timing_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s02/rafi_preclass_prep/timing_util.py -------------------------------------------------------------------------------- /s02/timing_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s02/timing_util.py -------------------------------------------------------------------------------- /s03/AfterSessionExercises.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s03/AfterSessionExercises.txt -------------------------------------------------------------------------------- /s03/Session3Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s03/Session3Slides.pdf -------------------------------------------------------------------------------- /s03/play_with_sharding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s03/play_with_sharding.py -------------------------------------------------------------------------------- /s03/play_with_sharding_in_class.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s03/play_with_sharding_in_class.py -------------------------------------------------------------------------------- /s04/AfterSessionExercises.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s04/AfterSessionExercises.txt -------------------------------------------------------------------------------- /s04/Session4Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s04/Session4Slides.pdf -------------------------------------------------------------------------------- /s04/before_class_allgather.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s04/before_class_allgather.py -------------------------------------------------------------------------------- /s04/before_class_matmuls.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s04/before_class_matmuls.py -------------------------------------------------------------------------------- /s04/in_class_all_gather.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s04/in_class_all_gather.py -------------------------------------------------------------------------------- /s04/in_class_matmul.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s04/in_class_matmul.py -------------------------------------------------------------------------------- /s04/shardings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s04/shardings.py -------------------------------------------------------------------------------- /s04/timing_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s04/timing_util.py -------------------------------------------------------------------------------- /s05/AfterSessionExercises.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s05/AfterSessionExercises.txt -------------------------------------------------------------------------------- /s05/Session5Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s05/Session5Slides.pdf -------------------------------------------------------------------------------- /s05/before_class_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s05/before_class_attention.py -------------------------------------------------------------------------------- /s05/during_class_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s05/during_class_attention.py -------------------------------------------------------------------------------- /s06/AfterSessionExercises.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s06/AfterSessionExercises.txt -------------------------------------------------------------------------------- /s06/Session6Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s06/Session6Slides.pdf -------------------------------------------------------------------------------- /s06/class_end.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s06/class_end.py -------------------------------------------------------------------------------- /s06/class_start.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s06/class_start.py -------------------------------------------------------------------------------- /s06/prep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s06/prep.py -------------------------------------------------------------------------------- /s07/AfterSessionExercises.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s07/AfterSessionExercises.txt -------------------------------------------------------------------------------- /s07/Session7Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s07/Session7Slides.pdf -------------------------------------------------------------------------------- /s07/class_end.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s07/class_end.py -------------------------------------------------------------------------------- /s07/class_start.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s07/class_start.py -------------------------------------------------------------------------------- /s07/prep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s07/prep.py -------------------------------------------------------------------------------- /s08/AfterSessionExercises.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s08/AfterSessionExercises.txt -------------------------------------------------------------------------------- /s08/Session8Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s08/Session8Slides.pdf -------------------------------------------------------------------------------- /s08/class_pretend_inference_is_training_prep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s08/class_pretend_inference_is_training_prep.py -------------------------------------------------------------------------------- /s08/class_pretend_inference_is_training_start.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s08/class_pretend_inference_is_training_start.py -------------------------------------------------------------------------------- /s08/class_softmax_class.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s08/class_softmax_class.py -------------------------------------------------------------------------------- /s08/class_softmax_primer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s08/class_softmax_primer.py -------------------------------------------------------------------------------- /s08/class_start.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s08/class_start.py -------------------------------------------------------------------------------- /s08/class_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s08/class_training.py -------------------------------------------------------------------------------- /s09/AfterSessionExercises.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s09/AfterSessionExercises.txt -------------------------------------------------------------------------------- /s09/Session9Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s09/Session9Slides.pdf -------------------------------------------------------------------------------- /s09/__pycache__/timing_util.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s09/__pycache__/timing_util.cpython-310.pyc -------------------------------------------------------------------------------- /s09/class_real_inference_end.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s09/class_real_inference_end.py -------------------------------------------------------------------------------- /s09/class_real_inference_prep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s09/class_real_inference_prep.py -------------------------------------------------------------------------------- /s09/class_real_inference_prep2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s09/class_real_inference_prep2.py -------------------------------------------------------------------------------- /s09/class_real_inference_start.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s09/class_real_inference_start.py -------------------------------------------------------------------------------- /s09/determinism_end.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s09/determinism_end.py -------------------------------------------------------------------------------- /s09/determinism_prep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s09/determinism_prep.py -------------------------------------------------------------------------------- /s09/determinism_start.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s09/determinism_start.py -------------------------------------------------------------------------------- /s09/timing_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s09/timing_util.py -------------------------------------------------------------------------------- /s10/AfterSessionExercises.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s10/AfterSessionExercises.txt -------------------------------------------------------------------------------- /s10/Session10Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rwitten/HighPerfLLMs2024/HEAD/s10/Session10Slides.pdf --------------------------------------------------------------------------------